From 9ff7b4af137b8028b04b52addf003c4b0607113b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 2 Aug 2023 14:40:23 +0200 Subject: [PATCH 001/598] gh-107559: Argument Clinic: complain about non-ASCII chars in param docstrings (#107560) Previously, only function docstrings were checked for non-ASCII characters. Also, improve the warn() message. Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 19 +++++++++++++++++++ Tools/clinic/clinic.py | 8 +++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6bdc571dd4d5ac..6f53036366891f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1427,6 +1427,25 @@ def test_scaffolding(self): actual = stdout.getvalue() self.assertEqual(actual, expected) + def test_non_ascii_character_in_docstring(self): + block = """ + module test + test.fn + a: int + á param docstring + docstring fü bár baß + """ + with support.captured_stdout() as stdout: + self.parse(block) + # The line numbers are off; this is a known limitation. + expected = dedent("""\ + Warning on line 0: + Non-ascii characters are not allowed in docstrings: 'á' + Warning on line 0: + Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' + """) + self.assertEqual(stdout.getvalue(), expected) + class ClinicExternalTest(TestCase): maxDiff = None diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5f7d41e4415515..1f461665003c81 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -785,9 +785,6 @@ def docstring_for_c_string( self, f: Function ) -> str: - if re.search(r'[^\x00-\x7F]', f.docstring): - warn("Non-ascii character appear in docstring.") - text, add, output = _text_accumulator() # turn docstring into a properly quoted C string for line in f.docstring.split('\n'): @@ -5266,6 +5263,11 @@ def state_parameter_docstring_start(self, line: str) -> None: def docstring_append(self, obj: Function | Parameter, line: str) -> None: """Add a rstripped line to the current docstring.""" + matches = re.finditer(r'[^\x00-\x7F]', line) + if offending := ", ".join([repr(m[0]) for m in matches]): + warn("Non-ascii characters are not allowed in docstrings:", + offending) + docstring = obj.docstring if docstring: docstring += "\n" From b9c9a36c2f2edc11b9c27eb7c5810919d9da9767 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 2 Aug 2023 15:33:10 +0200 Subject: [PATCH 002/598] gh-106368: Increase test coverage for Argument Clinic (#107514) As per this commit, we've got approx. ~91% test coverage for clinic.py. --- Lib/test/clinic.test.c | 260 ++++++++++++++++++++++++++++++++++++++++ Lib/test/test_clinic.py | 105 +++++++++++++--- 2 files changed, 350 insertions(+), 15 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index a660ccf8876e2d..a49c2e77810648 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5004,3 +5004,263 @@ PyDoc_STRVAR(new_dest__doc__, "\n" "Only this docstring should be outputted to test1."); /*[clinic end generated code: output=9cac703f51d90e84 input=090db8df4945576d]*/ + + +/*[clinic input] +mangled_c_keyword_identifier + i as int: int +The 'int' param should be mangled as 'int_value' +[clinic start generated code]*/ + +PyDoc_STRVAR(mangled_c_keyword_identifier__doc__, +"mangled_c_keyword_identifier($module, /, i)\n" +"--\n" +"\n" +"The \'int\' param should be mangled as \'int_value\'"); + +#define MANGLED_C_KEYWORD_IDENTIFIER_METHODDEF \ + {"mangled_c_keyword_identifier", _PyCFunction_CAST(mangled_c_keyword_identifier), METH_FASTCALL|METH_KEYWORDS, mangled_c_keyword_identifier__doc__}, + +static PyObject * +mangled_c_keyword_identifier_impl(PyObject *module, int int_value); + +static PyObject * +mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(i), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"i", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangled_c_keyword_identifier", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int int_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + int_value = _PyLong_AsInt(args[0]); + if (int_value == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = mangled_c_keyword_identifier_impl(module, int_value); + +exit: + return return_value; +} + +static PyObject * +mangled_c_keyword_identifier_impl(PyObject *module, int int_value) +/*[clinic end generated code: output=c049d7d79be26cda input=060876448ab567a2]*/ + + +/*[clinic input] +bool_return -> bool +[clinic start generated code]*/ + +PyDoc_STRVAR(bool_return__doc__, +"bool_return($module, /)\n" +"--\n" +"\n"); + +#define BOOL_RETURN_METHODDEF \ + {"bool_return", (PyCFunction)bool_return, METH_NOARGS, bool_return__doc__}, + +static int +bool_return_impl(PyObject *module); + +static PyObject * +bool_return(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = bool_return_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +static int +bool_return_impl(PyObject *module) +/*[clinic end generated code: output=3a65f07830e48e98 input=93ba95d39ee98f39]*/ + + +/*[clinic input] +double_return -> double +[clinic start generated code]*/ + +PyDoc_STRVAR(double_return__doc__, +"double_return($module, /)\n" +"--\n" +"\n"); + +#define DOUBLE_RETURN_METHODDEF \ + {"double_return", (PyCFunction)double_return, METH_NOARGS, double_return__doc__}, + +static double +double_return_impl(PyObject *module); + +static PyObject * +double_return(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + double _return_value; + + _return_value = double_return_impl(module); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +static double +double_return_impl(PyObject *module) +/*[clinic end generated code: output=076dc72595d3f66d input=da11b6255e4cbfd7]*/ + + +/*[clinic input] +Test.__init__ + a: object + [ + b: object + ] + / +Should generate two PyArg_ParseTuple calls. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(a, [b])\n" +"Should generate two PyArg_ParseTuple calls."); + +static int +Test___init___impl(TestObj *self, PyObject *a, int group_right_1, + PyObject *b); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *a; + int group_right_1 = 0; + PyObject *b = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:__init__", &a)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "OO:__init__", &a, &b)) { + goto exit; + } + group_right_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "Test.__init__ requires 1 to 2 arguments"); + goto exit; + } + return_value = Test___init___impl((TestObj *)self, a, group_right_1, b); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *a, int group_right_1, + PyObject *b) +/*[clinic end generated code: output=2bbb8ea60e8f57a6 input=10f5d0f1e8e466ef]*/ + + +/*[clinic input] +Test._pyarg_parsestackandkeywords + cls: defining_class + key: str(accept={str, robuffer}, zeroes=True) + / +Check that _PyArg_ParseStackAndKeywords() is generated. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__pyarg_parsestackandkeywords__doc__, +"_pyarg_parsestackandkeywords($self, key, /)\n" +"--\n" +"\n" +"Check that _PyArg_ParseStackAndKeywords() is generated."); + +#define TEST__PYARG_PARSESTACKANDKEYWORDS_METHODDEF \ + {"_pyarg_parsestackandkeywords", _PyCFunction_CAST(Test__pyarg_parsestackandkeywords), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test__pyarg_parsestackandkeywords__doc__}, + +static PyObject * +Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, + const char *key, + Py_ssize_t key_length); + +static PyObject * +Test__pyarg_parsestackandkeywords(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "s#:_pyarg_parsestackandkeywords", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + const char *key; + Py_ssize_t key_length; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &key, &key_length)) { + goto exit; + } + return_value = Test__pyarg_parsestackandkeywords_impl(self, cls, key, key_length); + +exit: + return return_value; +} + +static PyObject * +Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, + const char *key, + Py_ssize_t key_length) +/*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6f53036366891f..cdabcbaa6f03ca 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -8,6 +8,7 @@ from textwrap import dedent from unittest import TestCase import collections +import contextlib import inspect import os.path import sys @@ -335,6 +336,70 @@ def converter_init(self): ) self.assertIn(msg, out) + @staticmethod + @contextlib.contextmanager + def _clinic_version(new_version): + """Helper for test_version_*() tests""" + _saved = clinic.version + clinic.version = new_version + try: + yield + finally: + clinic.version = _saved + + def test_version_directive(self): + dataset = ( + # (clinic version, required version) + ('3', '2'), # required version < clinic version + ('3.1', '3.0'), # required version < clinic version + ('1.2b0', '1.2a7'), # required version < clinic version + ('5', '5'), # required version == clinic version + ('6.1', '6.1'), # required version == clinic version + ('1.2b3', '1.2b3'), # required version == clinic version + ) + for clinic_version, required_version in dataset: + with self.subTest(clinic_version=clinic_version, + required_version=required_version): + with self._clinic_version(clinic_version): + block = dedent(f""" + /*[clinic input] + version {required_version} + [clinic start generated code]*/ + """) + self.clinic.parse(block) + + def test_version_directive_insufficient_version(self): + with self._clinic_version('4'): + err = ( + "Insufficient Clinic version!\n" + " Version: 4\n" + " Required: 5" + ) + out = self.expect_failure(""" + /*[clinic input] + version 5 + [clinic start generated code]*/ + """) + self.assertIn(err, out) + + def test_version_directive_illegal_char(self): + err = "Illegal character 'v' in version string 'v5'" + out = self.expect_failure(""" + /*[clinic input] + version v5 + [clinic start generated code]*/ + """) + self.assertIn(err, out) + + def test_version_directive_unsupported_string(self): + err = "Unsupported version string: '.-'" + out = self.expect_failure(""" + /*[clinic input] + version .- + [clinic start generated code]*/ + """) + self.assertIn(err, out) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): @@ -513,6 +578,22 @@ def test_clinic_1(self): class ClinicParserTest(_ParserBase): + + def parse(self, text): + c = FakeClinic() + parser = DSLParser(c) + block = clinic.Block(text) + parser.parse(block) + return block + + def parse_function(self, text, signatures_in_block=2, function_index=1): + block = self.parse(text) + s = block.signatures + self.assertEqual(len(s), signatures_in_block) + assert isinstance(s[0], clinic.Module) + assert isinstance(s[function_index], clinic.Function) + return s[function_index] + def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) self.assertEqual(fn.docstring.strip(), @@ -1395,21 +1476,6 @@ def test_unused_param(self): parser_decl = p.simple_declaration(in_parser=True) self.assertNotIn("Py_UNUSED", parser_decl) - def parse(self, text): - c = FakeClinic() - parser = DSLParser(c) - block = clinic.Block(text) - parser.parse(block) - return block - - def parse_function(self, text, signatures_in_block=2, function_index=1): - block = self.parse(text) - s = block.signatures - self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) - return s[function_index] - def test_scaffolding(self): # test repr on special values self.assertEqual(repr(clinic.unspecified), '') @@ -1446,6 +1512,15 @@ def test_non_ascii_character_in_docstring(self): """) self.assertEqual(stdout.getvalue(), expected) + def test_illegal_c_identifier(self): + err = "Illegal C identifier: 17a" + out = self.parse_function_should_fail(""" + module test + test.fn + a as 17a: int + """) + self.assertIn(err, out) + class ClinicExternalTest(TestCase): maxDiff = None From dd693d6320feeca887174fa592537669d017ca9b Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 2 Aug 2023 18:16:57 +0100 Subject: [PATCH 003/598] gh-105481: simplify definition of pseudo ops in Lib/opcode.py (#107561) --- Doc/whatsnew/3.13.rst | 5 +++ Include/opcode.h | 2 - Lib/opcode.py | 44 +++++++------------ ...-08-01-21-43-58.gh-issue-105481.cl2ajS.rst | 2 + Tools/build/generate_opcode_h.py | 10 +---- 5 files changed, 23 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 22c1e03470f5dc..63cdee6cf1a4f3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -124,6 +124,11 @@ opcode This field was added in 3.12, it was never documented and is not intended for external usage. (Contributed by Irit Katriel in :gh:`105481`.) +* Removed ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and + ``opcode.MAX_PSEUDO_OPCODE``, which were added in 3.12, were never + documented or exposed through ``dis``, and were not intended to be + used externally. + pathlib ------- diff --git a/Include/opcode.h b/Include/opcode.h index eb4bb85e5b667d..ede1518b6fd25c 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -146,7 +146,6 @@ extern "C" { #define INSTRUMENTED_END_SEND 252 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 -#define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 #define SETUP_WITH 258 @@ -159,7 +158,6 @@ extern "C" { #define LOAD_ZERO_SUPER_ATTR 265 #define STORE_FAST_MAYBE_NULL 266 #define LOAD_CLOSURE 267 -#define MAX_PSEUDO_OPCODE 267 #define TO_BOOL_ALWAYS_TRUE 7 #define TO_BOOL_BOOL 8 #define TO_BOOL_INT 10 diff --git a/Lib/opcode.py b/Lib/opcode.py index 51432ab1fab3f1..5a9f8ddd0738db 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -19,23 +19,11 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - opmap = {} -# pseudo opcodes (used in the compiler) mapped to the values -# they can become in the actual code. -_pseudo_ops = {} - def def_op(name, op): opmap[name] = op -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - - # Instruction opcodes for compiled code # Blank lines correspond to available opcodes @@ -212,29 +200,27 @@ def pseudo_op(name, op, real_ops): # 255 is reserved -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -pseudo_op('SETUP_WITH', 258, ['NOP']) -pseudo_op('POP_BLOCK', 259, ['NOP']) +# Pseudo ops are above 255: -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) +def_op('SETUP_FINALLY', 256) +def_op('SETUP_CLEANUP', 257) +def_op('SETUP_WITH', 258) +def_op('POP_BLOCK', 259) -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) -pseudo_op('LOAD_SUPER_METHOD', 263, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR']) +def_op('JUMP', 260) +def_op('JUMP_NO_INTERRUPT', 261) -pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST']) -pseudo_op('LOAD_CLOSURE', 267, ['LOAD_FAST']) +def_op('LOAD_METHOD', 262) +def_op('LOAD_SUPER_METHOD', 263) +def_op('LOAD_ZERO_SUPER_METHOD', 264) +def_op('LOAD_ZERO_SUPER_ATTR', 265) -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 +def_op('STORE_FAST_MAYBE_NULL', 266) +def_op('LOAD_CLOSURE', 267) -del def_op, pseudo_op +del def_op -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] for op, i in opmap.items(): opname[i] = op diff --git a/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst b/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst new file mode 100644 index 00000000000000..d02f909e870188 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst @@ -0,0 +1,2 @@ +Remove ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and ``opcode.MAX_PSEUDO_OPCODE``, +which were added in 3.12, were never documented and were not intended to be used externally. diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 16b028dc1205ac..3a817326c94cbb 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -72,10 +72,7 @@ def main(opcode_py, opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] - is_pseudo = opcode['is_pseudo'] - MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] - MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] NUM_OPCODES = len(opname) @@ -101,16 +98,11 @@ def main(opcode_py, for name in opname: if name in opmap: op = opmap[name] - if op == MIN_PSEUDO_OPCODE: - fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) if op == MIN_INSTRUMENTED_OPCODE: fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) fobj.write(DEFINE.format(name, op)) - if op == MAX_PSEUDO_OPCODE: - fobj.write(DEFINE.format("MAX_PSEUDO_OPCODE", MAX_PSEUDO_OPCODE)) - for name, op in specialized_opmap.items(): fobj.write(DEFINE.format(name, op)) @@ -126,7 +118,7 @@ def main(opcode_py, deoptcodes = {} for basic, op in opmap.items(): - if not is_pseudo(op): + if op < 256: deoptcodes[basic] = basic for basic, family in _opcode_metadata["_specializations"].items(): for specialized in family: From 0d30a5a40968cce19750be78154232fae25d641f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 2 Aug 2023 18:44:20 +0100 Subject: [PATCH 004/598] GH-100964: Break cycles involving exception state when returning from generator (GH-107563) --- .../2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst | 2 ++ Objects/genobject.c | 9 ++++----- Python/ceval.c | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst new file mode 100644 index 00000000000000..99ebc926e2ce2d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst @@ -0,0 +1,2 @@ +Clear generators' exception state after ``return`` to break reference +cycles. diff --git a/Objects/genobject.c b/Objects/genobject.c index a630f84fb5a29d..65782be182cd71 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -149,14 +149,16 @@ gen_dealloc(PyGenObject *gen) gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); } + assert(gen->gi_exc_state.exc_value == NULL); if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); } Py_DECREF(_PyGen_GetCode(gen)); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); - _PyErr_ClearExcState(&gen->gi_exc_state); + PyObject_GC_Del(gen); } @@ -252,10 +254,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, !PyErr_ExceptionMatches(PyExc_StopAsyncIteration)); } - /* generator can't be rerun, so release the frame */ - /* first clean reference cycle through stored exception traceback */ - _PyErr_ClearExcState(&gen->gi_exc_state); - + assert(gen->gi_exc_state.exc_value == NULL); assert(gen->gi_frame_state == FRAME_CLEARED); *presult = result; return result ? PYGEN_RETURN : PYGEN_ERROR; diff --git a/Python/ceval.c b/Python/ceval.c index 17818a0d3c17e6..369b9a69152a5c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1466,6 +1466,7 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); tstate->c_recursion_remaining++; frame->previous = NULL; } From af8141cf879f606ed71da42e12f7ec0275f561db Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 2 Aug 2023 12:05:25 -0700 Subject: [PATCH 005/598] Fix test_capi.test_misc when run with -R:: (#107566) Should fix the buildbot failures. This creates a new function each time that test is run, like Victor did for other tests. --- Lib/test/test_capi/test_misc.py | 64 +++++++++++++++++---------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 0d3f93941e8205..e7cdd4be002a14 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2464,36 +2464,40 @@ def testfunc(x): def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 - while z9 > 0: - z9 = z9 - 1 + ns = {} + exec(textwrap.dedent(""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + while z9 > 0: + z9 = z9 - 1 + """), ns, ns) + many_vars = ns["many_vars"] opt = _testinternalcapi.get_uop_optimizer() with temporary_optimizer(opt): From bcdd3072316181b49d94567bb648825a07ca9ae1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 2 Aug 2023 21:37:36 +0200 Subject: [PATCH 006/598] gh-104683: Make Argument Clinic template strings class level members (#107556) The motivation for this change is to clean up the output_templates() method a little bit, as it accounts for ~10% of the lines of code in clinic.py; removing some clutter helps readability. --- Tools/clinic/clinic.py | 140 ++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 71 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1f461665003c81..ce8184753c7292 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -760,6 +760,59 @@ class CLanguage(Language): stop_line = "[{dsl_name} start generated code]*/" checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" + PARSER_PROTOTYPE_KEYWORD: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_VARARGS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args) + """) + PARSER_PROTOTYPE_FASTCALL: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) + """) + PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_DEF_CLASS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_NOARGS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) + """) + METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({impl_parameters}) + """) + DOCSTRING_PROTOTYPE_VAR: Final[str] = normalize_snippet(""" + PyDoc_VAR({c_basename}__doc__); + """) + DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" + PyDoc_STRVAR({c_basename}__doc__, + {docstring}); + """) + IMPL_DEFINITION_PROTOTYPE: Final[str] = normalize_snippet(""" + static {impl_return_type} + {c_basename}_impl({impl_parameters}) + """) + METHODDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + #define {methoddef_name} \ + {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, + """) + METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" + #ifndef {methoddef_name} + #define {methoddef_name} + #endif /* !defined({methoddef_name}) */ + """) + def __init__(self, filename: str) -> None: super().__init__(filename) self.cpp = cpp.Monitor(filename) @@ -862,52 +915,15 @@ def output_templates( # methoddef_ifndef return_value_declaration = "PyObject *return_value = NULL;" - - methoddef_define = normalize_snippet(""" - #define {methoddef_name} \\ - {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, - """) + methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE if new_or_init and not f.docstring: docstring_prototype = docstring_definition = '' else: - docstring_prototype = normalize_snippet(""" - PyDoc_VAR({c_basename}__doc__); - """) - docstring_definition = normalize_snippet(""" - PyDoc_STRVAR({c_basename}__doc__, - {docstring}); - """) - impl_definition = normalize_snippet(""" - static {impl_return_type} - {c_basename}_impl({impl_parameters}) - """) + docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR + docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR + impl_definition = self.IMPL_DEFINITION_PROTOTYPE impl_prototype = parser_prototype = parser_definition = None - parser_prototype_keyword = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - - parser_prototype_varargs = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args) - """) - - parser_prototype_fastcall = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) - """) - - parser_prototype_fastcall_keywords = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - - parser_prototype_def_class = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields: tuple[str, ...] = () @@ -950,19 +966,13 @@ def parser_body( if not requires_defining_class: # no parameters, METH_NOARGS flags = "METH_NOARGS" - - parser_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) - """) + parser_prototype = self.PARSER_PROTOTYPE_NOARGS parser_code = [] - else: assert not new_or_init flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" - - parser_prototype = parser_prototype_def_class + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS return_error = ('return NULL;' if default_return_converter else 'goto exit;') parser_code = [normalize_snippet(""" @@ -987,10 +997,7 @@ def parser_body( if (isinstance(converters[0], object_converter) and converters[0].format_unit == 'O'): - meth_o_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({impl_parameters}) - """) + meth_o_prototype = self.METH_O_PROTOTYPE if default_return_converter: # maps perfectly to METH_O, doesn't need a return converter. @@ -1030,8 +1037,7 @@ def parser_body( # in a big switch statement) flags = "METH_VARARGS" - parser_prototype = parser_prototype_varargs - + parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: @@ -1040,7 +1046,7 @@ def parser_body( # we only need one call to _PyArg_ParseStack flags = "METH_FASTCALL" - parser_prototype = parser_prototype_fastcall + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL nargs = 'nargs' argname_fmt = 'args[%d]' else: @@ -1048,7 +1054,7 @@ def parser_body( # we only need one call to PyArg_ParseTuple flags = "METH_VARARGS" - parser_prototype = parser_prototype_varargs + parser_prototype = self.PARSER_PROTOTYPE_VARARGS nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' @@ -1145,7 +1151,7 @@ def parser_body( nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = parser_prototype_fastcall_keywords + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS argname_fmt = 'args[%d]' declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) @@ -1160,7 +1166,7 @@ def parser_body( else: # positional-or-keyword arguments flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = parser_prototype_keyword + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD argname_fmt = 'fastargs[%d]' declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) @@ -1177,7 +1183,7 @@ def parser_body( if requires_defining_class: flags = 'METH_METHOD|' + flags - parser_prototype = parser_prototype_def_class + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS add_label: str | None = None for i, p in enumerate(parameters): @@ -1264,13 +1270,10 @@ def parser_body( methoddef_define = '' if f.kind is METHOD_NEW: - parser_prototype = parser_prototype_keyword + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD else: return_value_declaration = "int return_value = -1;" - parser_prototype = normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ fields = list(parser_body_fields) parses_positional = 'METH_NOARGS' not in flags @@ -1323,12 +1326,7 @@ def parser_body( if methoddef_define and f.full_name not in clinic.ifndef_symbols: clinic.ifndef_symbols.add(f.full_name) - methoddef_ifndef = normalize_snippet(""" - #ifndef {methoddef_name} - #define {methoddef_name} - #endif /* !defined({methoddef_name}) */ - """) - + methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF # add ';' to the end of parser_prototype and impl_prototype # (they mustn't be None, but they could be an empty string.) From 017f047183fa33743f7e36c5c360f5c670032be3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 2 Aug 2023 14:55:09 -0600 Subject: [PATCH 007/598] gh-107471: Fix Refleaks in test_import (gh-107569) gh-107184 introduced a refleak in test_import.SubinterpImportTests (specifically test_singlephase_check_with_setting_and_override and test_single_init_extension_compat). We fix it here by making sure _testsinglephase is removed from sys.modules whenever we clear the runtime's internal state for the module. The underlying problem is strictly contained in the internal function _PyImport_ClearExtension() (AKA _testinternalcapi.clear_extension()), which is only used in tests. (This also fixes an intermittent segfault introduced in the same place, in test_disallowed_reimport.) --- Lib/test/test_import/__init__.py | 3 ++- Python/import.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7a3fcc22be8d13..163ed824ff1fb0 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -150,6 +150,7 @@ def _ready_to_import(name=None, source=""): def restore__testsinglephase(*, _orig=_testsinglephase): # We started with the module imported and want to restore # it to its nominal state. + sys.modules.pop('_testsinglephase', None) _orig._clear_globals() _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) import _testsinglephase @@ -2125,7 +2126,7 @@ def clean_up(): _interpreters.run_string(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: - sys.modules[name]._clear_globals() + sys.modules.pop(name)._clear_globals() _testinternalcapi.clear_extension(name, {self.FILE!r}) ''')) _interpreters.destroy(interpid) diff --git a/Python/import.c b/Python/import.c index 3be2f76c9eca75..56b2dc1a4ada2c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1073,6 +1073,7 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) However, this decref would be problematic if the module def were dynamically allocated, it were the last ref, and this function were called with an interpreter other than the def's owner. */ + assert(_Py_IsImmortal(entry->value)); entry->value = NULL; finally: From 1cd479c6d371605e9689c88ae1789dbcbceb2da0 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 3 Aug 2023 02:00:06 +0200 Subject: [PATCH 008/598] gh-104683: Rework Argument Clinic error handling (#107551) Introduce ClinicError, and use it in fail(). The CLI runs main(), catches ClinicError, formats the error message, prints to stderr and exits with an error. As a side effect, this refactor greatly improves the accuracy of reported line numbers in case of error. Also, adapt the test suite to work with ClinicError. Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 499 ++++++++++++++++++---------------------- Tools/clinic/clinic.py | 69 +++--- 2 files changed, 270 insertions(+), 298 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index cdabcbaa6f03ca..127008d443e4c6 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -11,6 +11,7 @@ import contextlib import inspect import os.path +import re import sys import unittest @@ -20,17 +21,24 @@ from clinic import DSLParser -class _ParserBase(TestCase): - maxDiff = None - - def expect_parser_failure(self, parser, _input): - with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - parser(_input) - return stdout.getvalue() +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): + """Helper for the parser tests. - def parse_function_should_fail(self, _input): - return self.expect_parser_failure(self.parse_function, _input) + tc: unittest.TestCase; passed self in the wrapper + parser: the clinic parser used for this test case + code: a str with input text (clinic code) + errmsg: the expected error message + filename: str, optional filename + lineno: int, optional line number + """ + code = dedent(code).strip() + errmsg = re.escape(errmsg) + with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + parser(code) + if filename is not None: + tc.assertEqual(cm.exception.filename, filename) + if lineno is not None: + tc.assertEqual(cm.exception.lineno, lineno) class FakeConverter: @@ -108,14 +116,15 @@ def __repr__(self): return "" -class ClinicWholeFileTest(_ParserBase): +class ClinicWholeFileTest(TestCase): + + def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): + _expect_failure(self, self.clinic.parse, raw, errmsg, + filename=filename, lineno=lineno) + def setUp(self): self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") - def expect_failure(self, raw): - _input = dedent(raw).strip() - return self.expect_parser_failure(self.clinic.parse, _input) - def test_eol(self): # regression test: # clinic's block parser didn't recognize @@ -139,12 +148,11 @@ def test_mangled_marker_line(self): [clinic start generated code]*/ /*[clinic end generated code: foo]*/ """ - msg = ( - 'Error in file "test.c" on line 3:\n' - "Mangled Argument Clinic marker line: '/*[clinic end generated code: foo]*/'\n" + err = ( + "Mangled Argument Clinic marker line: " + "'/*[clinic end generated code: foo]*/'" ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + self.expect_failure(raw, err, filename="test.c", lineno=3) def test_checksum_mismatch(self): raw = """ @@ -152,38 +160,31 @@ def test_checksum_mismatch(self): [clinic start generated code]*/ /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ """ - msg = ( - 'Error in file "test.c" on line 3:\n' + err = ( 'Checksum mismatch!\n' 'Expected: 0123456789abcdef\n' 'Computed: da39a3ee5e6b4b0d\n' ) - out = self.expect_failure(raw) - self.assertIn(msg, out) + self.expect_failure(raw, err, filename="test.c", lineno=3) def test_garbage_after_stop_line(self): raw = """ /*[clinic input] [clinic start generated code]*/foobarfoobar! """ - msg = ( - 'Error in file "test.c" on line 2:\n' - "Garbage after stop line: 'foobarfoobar!'\n" - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = "Garbage after stop line: 'foobarfoobar!'" + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_whitespace_before_stop_line(self): raw = """ /*[clinic input] [clinic start generated code]*/ """ - msg = ( - 'Error in file "test.c" on line 2:\n' - "Whitespace is not allowed before the stop line: ' [clinic start generated code]*/'\n" + err = ( + "Whitespace is not allowed before the stop line: " + "' [clinic start generated code]*/'" ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): clang = clinic.CLanguage(None) @@ -213,12 +214,8 @@ def test_cpp_monitor_fail_nested_block_comment(self): */ */ """ - msg = ( - 'Error in file "test.c" on line 2:\n' - 'Nested block comment!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Nested block comment!' + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_cpp_monitor_fail_invalid_format_noarg(self): raw = """ @@ -226,12 +223,8 @@ def test_cpp_monitor_fail_invalid_format_noarg(self): a() #endif """ - msg = ( - 'Error in file "test.c" on line 1:\n' - 'Invalid format for #if line: no argument!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Invalid format for #if line: no argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_cpp_monitor_fail_invalid_format_toomanyargs(self): raw = """ @@ -239,39 +232,31 @@ def test_cpp_monitor_fail_invalid_format_toomanyargs(self): a() #endif """ - msg = ( - 'Error in file "test.c" on line 1:\n' - 'Invalid format for #ifdef line: should be exactly one argument!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Invalid format for #ifdef line: should be exactly one argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_cpp_monitor_fail_no_matching_if(self): raw = '#else' - msg = ( - 'Error in file "test.c" on line 1:\n' - '#else without matching #if / #ifdef / #ifndef!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = '#else without matching #if / #ifdef / #ifndef!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_directive_output_unknown_preset(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] output preset nosuchpreset [clinic start generated code]*/ - """) - msg = "Unknown preset 'nosuchpreset'" - self.assertIn(msg, out) + """ + err = "Unknown preset 'nosuchpreset'" + self.expect_failure(raw, err) def test_directive_output_cant_pop(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] output pop [clinic start generated code]*/ - """) - msg = "Can't 'output pop', stack is empty" - self.assertIn(msg, out) + """ + err = "Can't 'output pop', stack is empty" + self.expect_failure(raw, err) def test_directive_output_print(self): raw = dedent(""" @@ -309,16 +294,16 @@ def test_directive_output_print(self): ) def test_unknown_destination_command(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] destination buffer nosuchcommand [clinic start generated code]*/ - """) - msg = "unknown destination command 'nosuchcommand'" - self.assertIn(msg, out) + """ + err = "unknown destination command 'nosuchcommand'" + self.expect_failure(raw, err) def test_no_access_to_members_in_converter_init(self): - out = self.expect_failure(""" + raw = """ /*[python input] class Custom_converter(CConverter): converter = "some_c_function" @@ -330,11 +315,11 @@ def converter_init(self): test.fn a: Custom [clinic start generated code]*/ - """) - msg = ( + """ + err = ( "accessing self.function inside converter_init is disallowed!" ) - self.assertIn(msg, out) + self.expect_failure(raw, err) @staticmethod @contextlib.contextmanager @@ -375,30 +360,30 @@ def test_version_directive_insufficient_version(self): " Version: 4\n" " Required: 5" ) - out = self.expect_failure(""" + block = """ /*[clinic input] version 5 [clinic start generated code]*/ - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) def test_version_directive_illegal_char(self): err = "Illegal character 'v' in version string 'v5'" - out = self.expect_failure(""" + block = """ /*[clinic input] version v5 [clinic start generated code]*/ - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) def test_version_directive_unsupported_string(self): err = "Unsupported version string: '.-'" - out = self.expect_failure(""" + block = """ /*[clinic input] version .- [clinic start generated code]*/ - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) class ClinicGroupPermuterTest(TestCase): @@ -577,7 +562,7 @@ def test_clinic_1(self): """) -class ClinicParserTest(_ParserBase): +class ClinicParserTest(TestCase): def parse(self, text): c = FakeClinic() @@ -594,6 +579,10 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): assert isinstance(s[function_index], clinic.Function) return s[function_index] + def expect_failure(self, block, err, *, filename=None, lineno=None): + _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno) + def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) self.assertEqual(fn.docstring.strip(), @@ -663,17 +652,16 @@ def test_param_default_expression(self): self.assertEqual(sys.maxsize, p.default) self.assertEqual("MAXSIZE", p.converter.c_default) - expected_msg = ( - "Error on line 0:\n" + err = ( "When you specify a named constant ('sys.maxsize') as your default value,\n" - "you MUST specify a valid c_default.\n" + "you MUST specify a valid c_default." ) - out = self.parse_function_should_fail(""" + block = """ module os os.access follow_symlinks: int = sys.maxsize - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=2) def test_param_no_docstring(self): function = self.parse_function(""" @@ -688,17 +676,17 @@ def test_param_no_docstring(self): self.assertIsInstance(conv, clinic.str_converter) def test_param_default_parameters_out_of_order(self): - expected_msg = ( - "Error on line 0:\n" + err = ( "Can't have a parameter without a default ('something_else')\n" - "after a parameter with a default!\n" + "after a parameter with a default!" ) - out = self.parse_function_should_fail(""" + block = """ module os os.access follow_symlinks: bool = True - something_else: str""") - self.assertEqual(out, expected_msg) + something_else: str + """ + self.expect_failure(block, err, lineno=3) def disabled_test_converter_arguments(self): function = self.parse_function(""" @@ -797,17 +785,19 @@ def test_c_name(self): self.assertEqual("os_stat_fn", function.c_basename) def test_cloning_nonexistent_function_correctly_fails(self): - stdout = self.parse_function_should_fail(""" - cloned = fooooooooooooooooooooooo + block = """ + cloned = fooooooooooooooooo This is trying to clone a nonexistent function!! + """ + err = "Couldn't find existing function 'fooooooooooooooooo'!" + with support.captured_stderr() as stderr: + self.expect_failure(block, err, lineno=0) + expected_debug_print = dedent("""\ + cls=None, module=, existing='fooooooooooooooooo' + (cls or module).functions=[] """) - expected_error = """\ -cls=None, module=, existing='fooooooooooooooooooooooo' -(cls or module).functions=[] -Error on line 0: -Couldn't find existing function 'fooooooooooooooooooooooo'! -""" - self.assertEqual(expected_error, stdout) + stderr = stderr.getvalue() + self.assertIn(expected_debug_print, stderr) def test_return_converter(self): function = self.parse_function(""" @@ -817,30 +807,28 @@ def test_return_converter(self): self.assertIsInstance(function.return_converter, clinic.int_return_converter) def test_return_converter_invalid_syntax(self): - stdout = self.parse_function_should_fail(""" + block = """ module os os.stat -> invalid syntax - """) - expected_error = "Badly formed annotation for os.stat: 'invalid syntax'" - self.assertIn(expected_error, stdout) + """ + err = "Badly formed annotation for os.stat: 'invalid syntax'" + self.expect_failure(block, err) def test_legacy_converter_disallowed_in_return_annotation(self): - stdout = self.parse_function_should_fail(""" + block = """ module os os.stat -> "s" - """) - expected_error = "Legacy converter 's' not allowed as a return converter" - self.assertIn(expected_error, stdout) + """ + err = "Legacy converter 's' not allowed as a return converter" + self.expect_failure(block, err) def test_unknown_return_converter(self): - stdout = self.parse_function_should_fail(""" + block = """ module os - os.stat -> foooooooooooooooooooooooo - """) - expected_error = ( - "No available return converter called 'foooooooooooooooooooooooo'" - ) - self.assertIn(expected_error, stdout) + os.stat -> fooooooooooooooooooooooo + """ + err = "No available return converter called 'fooooooooooooooooooooooo'" + self.expect_failure(block, err) def test_star(self): function = self.parse_function(""" @@ -985,19 +973,12 @@ def test_nested_groups(self): Attributes for the character. """) - def parse_function_should_fail(self, s): - with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - self.parse_function(s) - return stdout.getvalue() - def test_disallowed_grouping__two_top_groups_on_left(self): - expected_msg = ( - 'Error on line 0:\n' + err = ( 'Function two_top_groups_on_left has an unsupported group ' - 'configuration. (Unexpected state 2.b)\n' + 'configuration. (Unexpected state 2.b)' ) - out = self.parse_function_should_fail(""" + block = """ module foo foo.two_top_groups_on_left [ @@ -1007,11 +988,11 @@ def test_disallowed_grouping__two_top_groups_on_left(self): group2 : int ] param: int - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=5) def test_disallowed_grouping__two_top_groups_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.two_top_groups_on_right param: int @@ -1021,15 +1002,15 @@ def test_disallowed_grouping__two_top_groups_on_right(self): [ group2 : int ] - """) - msg = ( + """ + err = ( "Function two_top_groups_on_right has an unsupported group " "configuration. (Unexpected state 6.b)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__parameter_after_group_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.parameter_after_group_on_right param: int @@ -1039,15 +1020,15 @@ def test_disallowed_grouping__parameter_after_group_on_right(self): ] group2 : int ] - """) - msg = ( + """ + err = ( "Function parameter_after_group_on_right has an unsupported group " "configuration. (Unexpected state 6.a)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__group_after_parameter_on_left(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.group_after_parameter_on_left [ @@ -1057,15 +1038,15 @@ def test_disallowed_grouping__group_after_parameter_on_left(self): ] ] param: int - """) - msg = ( + """ + err = ( "Function group_after_parameter_on_left has an unsupported group " "configuration. (Unexpected state 2.b)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__empty_group_on_left(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group [ @@ -1074,15 +1055,15 @@ def test_disallowed_grouping__empty_group_on_left(self): group2 : int ] param: int - """) - msg = ( + """ + err = ( "Function empty_group has an empty group.\n" "All groups must contain at least one parameter." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__empty_group_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group param: int @@ -1091,24 +1072,24 @@ def test_disallowed_grouping__empty_group_on_right(self): ] group2 : int ] - """) - msg = ( + """ + err = ( "Function empty_group has an empty group.\n" "All groups must contain at least one parameter." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__no_matching_bracket(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group param: int ] group2: int ] - """) - msg = "Function empty_group has a ] without a matching [." - self.assertIn(msg, out) + """ + err = "Function empty_group has a ] without a matching [." + self.expect_failure(block, err) def test_no_parameters(self): function = self.parse_function(""" @@ -1137,31 +1118,32 @@ class foo.Bar "unused" "notneeded" self.assertEqual(1, len(function.parameters)) def test_illegal_module_line(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar => int / - """) - msg = "Illegal function name: foo.bar => int" - self.assertIn(msg, out) + """ + err = "Illegal function name: foo.bar => int" + self.expect_failure(block, err) def test_illegal_c_basename(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar as 935 / - """) - msg = "Illegal C basename: 935" - self.assertIn(msg, out) + """ + err = "Illegal C basename: 935" + self.expect_failure(block, err) def test_single_star(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar * * - """) - self.assertIn("Function bar uses '*' more than once.", out) + """ + err = "Function bar uses '*' more than once." + self.expect_failure(block, err) def test_parameters_required_after_star(self): dataset = ( @@ -1170,39 +1152,38 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - msg = "Function bar specifies '*' without any parameters afterwards." + err = "Function bar specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_single_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar / / - """) - msg = ( + """ + err = ( "Function bar has an unsupported group configuration. " "(Unexpected state 0.d)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_double_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar a: int / b: int / - """) - msg = "Function bar uses '/' more than once." - self.assertIn(msg, out) + """ + err = "Function bar uses '/' more than once." + self.expect_failure(block, err) def test_mix_star_and_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar x: int @@ -1210,38 +1191,35 @@ def test_mix_star_and_slash(self): * z: int / - """) - msg = ( + """ + err = ( "Function bar mixes keyword-only and positional-only parameters, " "which is unsupported." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_parameters_not_permitted_after_slash_for_now(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar / x: int - """) - msg = ( + """ + err = ( "Function bar has an unsupported group configuration. " "(Unexpected state 0.d)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_parameters_no_more_than_one_vararg(self): - expected_msg = ( - "Error on line 0:\n" - "Too many var args\n" - ) - out = self.parse_function_should_fail(""" + err = "Too many var args" + block = """ module foo foo.bar *vararg1: object *vararg2: object - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=0) def test_function_not_at_column_0(self): function = self.parse_function(""" @@ -1264,23 +1242,24 @@ def test_function_not_at_column_0(self): """) def test_indent_stack_no_tabs(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar *vararg1: object \t*vararg2: object - """) - msg = "Tab characters are illegal in the Clinic DSL." - self.assertIn(msg, out) + """ + err = "Tab characters are illegal in the Clinic DSL." + self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar a: object b: object - """) - self.assertIn("Illegal outdent", out) + """ + err = "Illegal outdent" + self.expect_failure(block, err) def test_directive(self): c = FakeClinic() @@ -1298,10 +1277,7 @@ def test_legacy_converters(self): self.assertIsInstance(conv, clinic.str_converter) def test_legacy_converters_non_string_constant_annotation(self): - expected_failure_message = ( - "Error on line 0:\n" - "Annotations must be either a name, a function call, or a string.\n" - ) + err = "Annotations must be either a name, a function call, or a string" dataset = ( 'module os\nos.access\n path: 42', 'module os\nos.access\n path: 42.42', @@ -1310,14 +1286,10 @@ def test_legacy_converters_non_string_constant_annotation(self): ) for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_failure_message) + self.expect_failure(block, err, lineno=2) def test_other_bizarre_things_in_annotations_fail(self): - expected_failure_message = ( - "Error on line 0:\n" - "Annotations must be either a name, a function call, or a string.\n" - ) + err = "Annotations must be either a name, a function call, or a string" dataset = ( 'module os\nos.access\n path: {"some": "dictionary"}', 'module os\nos.access\n path: ["list", "of", "strings"]', @@ -1325,30 +1297,24 @@ def test_other_bizarre_things_in_annotations_fail(self): ) for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_failure_message) + self.expect_failure(block, err, lineno=2) def test_kwarg_splats_disallowed_in_function_call_annotations(self): - expected_error_msg = ( - "Error on line 0:\n" - "Cannot use a kwarg splat in a function-call annotation\n" - ) + err = "Cannot use a kwarg splat in a function-call annotation" dataset = ( 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', 'module fo\nfo.barbaz -> bool(**{"bang": 42})', 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', ) - for fn in dataset: - with self.subTest(fn=fn): - out = self.parse_function_should_fail(fn) - self.assertEqual(out, expected_error_msg) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) def test_self_param_placement(self): - expected_error_msg = ( - "Error on line 0:\n" + err = ( "A 'self' parameter, if specified, must be the very first thing " - "in the parameter block.\n" + "in the parameter block." ) block = """ module foo @@ -1356,27 +1322,21 @@ def test_self_param_placement(self): a: int self: self(type="PyObject *") """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=3) def test_self_param_cannot_be_optional(self): - expected_error_msg = ( - "Error on line 0:\n" - "A 'self' parameter cannot be marked optional.\n" - ) + err = "A 'self' parameter cannot be marked optional." block = """ module foo foo.func self: self(type="PyObject *") = None """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=2) def test_defining_class_param_placement(self): - expected_error_msg = ( - "Error on line 0:\n" + err = ( "A 'defining_class' parameter, if specified, must either be the " - "first thing in the parameter block, or come just after 'self'.\n" + "first thing in the parameter block, or come just after 'self'." ) block = """ module foo @@ -1385,21 +1345,16 @@ def test_defining_class_param_placement(self): a: int cls: defining_class """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=4) def test_defining_class_param_cannot_be_optional(self): - expected_error_msg = ( - "Error on line 0:\n" - "A 'defining_class' parameter cannot be marked optional.\n" - ) + err = "A 'defining_class' parameter cannot be marked optional." block = """ module foo foo.func cls: defining_class(type="PyObject *") = None """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=2) def test_slot_methods_cannot_access_defining_class(self): block = """ @@ -1409,34 +1364,28 @@ class Foo "" "" cls: defining_class a: object """ - msg = "Slot methods cannot access their defining class." - with self.assertRaisesRegex(ValueError, msg): + err = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, err): self.parse_function(block) def test_new_must_be_a_class_method(self): - expected_error_msg = ( - "Error on line 0:\n" - "__new__ must be a class method!\n" - ) - out = self.parse_function_should_fail(""" + err = "__new__ must be a class method!" + block = """ module foo class Foo "" "" Foo.__new__ - """) - self.assertEqual(out, expected_error_msg) + """ + self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - expected_error_msg = ( - "Error on line 0:\n" - "__init__ must be a normal method, not a class or static method!\n" - ) - out = self.parse_function_should_fail(""" + err = "__init__ must be a normal method, not a class or static method!" + block = """ module foo class Foo "" "" @classmethod Foo.__init__ - """) - self.assertEqual(out, expected_error_msg) + """ + self.expect_failure(block, err, lineno=3) def test_unused_param(self): block = self.parse(""" @@ -1487,11 +1436,12 @@ def test_scaffolding(self): 'The igloos are melting!\n' ) with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - clinic.fail('The igloos are melting!', - filename='clown.txt', line_number=69) - actual = stdout.getvalue() - self.assertEqual(actual, expected) + errmsg = 'The igloos are melting' + with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + clinic.fail(errmsg, filename='clown.txt', line_number=69) + exc = cm.exception + self.assertEqual(exc.filename, 'clown.txt') + self.assertEqual(exc.lineno, 69) def test_non_ascii_character_in_docstring(self): block = """ @@ -1507,19 +1457,21 @@ def test_non_ascii_character_in_docstring(self): expected = dedent("""\ Warning on line 0: Non-ascii characters are not allowed in docstrings: 'á' + Warning on line 0: Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' + """) self.assertEqual(stdout.getvalue(), expected) def test_illegal_c_identifier(self): err = "Illegal C identifier: 17a" - out = self.parse_function_should_fail(""" + block = """ module test test.fn a as 17a: int - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) class ClinicExternalTest(TestCase): @@ -1607,9 +1559,9 @@ def test_cli_force(self): # First, run the CLI without -f and expect failure. # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. - out, _ = self.expect_failure(fn) - self.assertTrue(out.endswith(fail_msg), - f"{out!r} does not end with {fail_msg!r}") + _, err = self.expect_failure(fn) + self.assertTrue(err.endswith(fail_msg), + f"{err!r} does not end with {fail_msg!r}") # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -2231,8 +2183,11 @@ def test_gh_99233_refcount(self): self.assertEqual(arg_refcount_origin, arg_refcount_after) def test_gh_99240_double_free(self): - expected_error = r'gh_99240_double_free\(\) argument 2 must be encoded string without null bytes, not str' - with self.assertRaisesRegex(TypeError, expected_error): + err = re.escape( + "gh_99240_double_free() argument 2 must be encoded string " + "without null bytes, not str" + ) + with self.assertRaisesRegex(TypeError, err): ac_tester.gh_99240_double_free('a', '\0b') def test_cloned_func_exception_message(self): diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ce8184753c7292..1bcdb6b1c3640a 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -28,7 +28,6 @@ import string import sys import textwrap -import traceback from collections.abc import ( Callable, @@ -137,6 +136,28 @@ def text_accumulator() -> TextAccumulator: text, append, output = _text_accumulator() return TextAccumulator(append, output) + +@dc.dataclass +class ClinicError(Exception): + message: str + _: dc.KW_ONLY + lineno: int | None = None + filename: str | None = None + + def __post_init__(self) -> None: + super().__init__(self.message) + + def report(self, *, warn_only: bool = False) -> str: + msg = "Warning" if warn_only else "Error" + if self.filename is not None: + msg += f" in file {self.filename!r}" + if self.lineno is not None: + msg += f" on line {self.lineno}" + msg += ":\n" + msg += f"{self.message}\n" + return msg + + @overload def warn_or_fail( *args: object, @@ -160,25 +181,16 @@ def warn_or_fail( line_number: int | None = None, ) -> None: joined = " ".join([str(a) for a in args]) - add, output = text_accumulator() - if fail: - add("Error") - else: - add("Warning") if clinic: if filename is None: filename = clinic.filename if getattr(clinic, 'block_parser', None) and (line_number is None): line_number = clinic.block_parser.line_number - if filename is not None: - add(' in file "' + filename + '"') - if line_number is not None: - add(" on line " + str(line_number)) - add(':\n') - add(joined) - print(output()) + error = ClinicError(joined, filename=filename, lineno=line_number) if fail: - sys.exit(-1) + raise error + else: + print(error.report(warn_only=True)) def warn( @@ -347,7 +359,7 @@ def version_splitter(s: str) -> tuple[int, ...]: accumulator: list[str] = [] def flush() -> None: if not accumulator: - raise ValueError('Unsupported version string: ' + repr(s)) + fail(f'Unsupported version string: {s!r}') version.append(int(''.join(accumulator))) accumulator.clear() @@ -360,7 +372,7 @@ def flush() -> None: flush() version.append('abc'.index(c) - 3) else: - raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) + fail(f'Illegal character {c!r} in version string {s!r}') flush() return tuple(version) @@ -2233,11 +2245,7 @@ def parse(self, input: str) -> str: assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." self.parsers[dsl_name] = parsers[dsl_name](self) parser = self.parsers[dsl_name] - try: - parser.parse(block) - except Exception: - fail('Exception raised during parsing:\n' + - traceback.format_exc().rstrip()) + parser.parse(block) printer.print_block(block) # these are destinations not buffers @@ -4600,7 +4608,11 @@ def parse(self, block: Block) -> None: for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): if '\t' in line: fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) - self.state(line) + try: + self.state(line) + except ClinicError as exc: + exc.lineno = line_number + raise self.do_post_block_processing_cleanup() block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) @@ -4701,8 +4713,8 @@ def state_modulename_name(self, line: str) -> None: if existing_function.name == function_name: break else: - print(f"{cls=}, {module=}, {existing=}") - print(f"{(cls or module).functions=}") + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) fail(f"Couldn't find existing function {existing!r}!") fields = [x.strip() for x in full_name.split('.')] @@ -5719,8 +5731,13 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: def main(argv: list[str] | None = None) -> NoReturn: parser = create_cli() args = parser.parse_args(argv) - run_clinic(parser, args) - sys.exit(0) + try: + run_clinic(parser, args) + except ClinicError as exc: + sys.stderr.write(exc.report()) + sys.exit(1) + else: + sys.exit(0) if __name__ == "__main__": From 62a3a15119cf16d216a2cc7dc10d97143017ef62 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Thu, 3 Aug 2023 07:29:03 +0200 Subject: [PATCH 009/598] gh-107455: ctypes: Improve error messages when converting to an incompatible type (#107456) --- Lib/test/test_ctypes/test_functions.py | 52 ++++++++++++++++++- ...-07-30-14-18-49.gh-issue-107455.Es53l7.rst | 3 ++ Modules/_ctypes/_ctypes.c | 18 +++---- 3 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 9cf680f16620ac..08eecbc9ea4442 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -4,8 +4,8 @@ import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, byref, POINTER, pointer, ArgumentError, - c_char, c_wchar, c_byte, c_char_p, - c_short, c_int, c_long, c_longlong, + c_char, c_wchar, c_byte, c_char_p, c_wchar_p, + c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) from _ctypes import _Pointer, _SimpleCData @@ -92,6 +92,54 @@ def test_wchar_parm(self): "argument 2: TypeError: one character unicode string " "expected") + def test_c_char_p_parm(self): + """Test the error message when converting an incompatible type to c_char_p.""" + proto = CFUNCTYPE(c_int, c_char_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(b"abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_char_p") + + def test_c_wchar_p_parm(self): + """Test the error message when converting an incompatible type to c_wchar_p.""" + proto = CFUNCTYPE(c_int, c_wchar_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback("abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_wchar_p") + + def test_c_void_p_parm(self): + """Test the error message when converting an incompatible type to c_void_p.""" + proto = CFUNCTYPE(c_int, c_void_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(5), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(2.5) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'float' object cannot be " + "interpreted as ctypes.c_void_p") + def test_wchar_result(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst new file mode 100644 index 00000000000000..84a93251e799d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst @@ -0,0 +1,3 @@ +Improve error messages when converting an incompatible type to +:class:`ctypes.c_char_p`, :class:`ctypes.c_wchar_p` and +:class:`ctypes.c_void_p`. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8d4fd30150f19c..9aee37a9d954ef 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1728,9 +1728,9 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_wchar_p", Py_TYPE(value)->tp_name); return NULL; } @@ -1792,9 +1792,9 @@ c_char_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_char_p", Py_TYPE(value)->tp_name); return NULL; } @@ -1927,9 +1927,9 @@ c_void_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_void_p", Py_TYPE(value)->tp_name); return NULL; } From 46366ca0486d07fe94c70d00771482c8ef1546fc Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 3 Aug 2023 12:06:02 +0530 Subject: [PATCH 010/598] GH-107458: fix test_tools refleak (#107577) --- Lib/test/test_tools/test_sundry.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 2f8ba272164d32..d0b702d392cdf6 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -19,17 +19,11 @@ class TestSundryScripts(unittest.TestCase): # cleanly the logging module. @import_helper.mock_register_at_fork def test_sundry(self, mock_os): - old_modules = import_helper.modules_setup() - try: - for fn in os.listdir(scriptsdir): - if not fn.endswith('.py'): - continue - - name = fn[:-3] - import_tool(name) - finally: - # Unload all modules loaded in this test - import_helper.modules_cleanup(*old_modules) + for fn in os.listdir(scriptsdir): + if not fn.endswith('.py'): + continue + name = fn[:-3] + import_tool(name) if __name__ == '__main__': From a73daf54ebd7bd6bf32e82766a605ebead2f128c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 3 Aug 2023 11:35:26 +0200 Subject: [PATCH 011/598] gh-106368: Increase Argument Clinic test coverage (#107582) Add tests for DSL parser state machine and docstring formatting --- Lib/test/clinic.test.c | 200 ++++++++++++++++++++++++++++++++++++++++ Lib/test/test_clinic.py | 196 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 394 insertions(+), 2 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index a49c2e77810648..d2ad1a0482c304 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5264,3 +5264,203 @@ Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) /*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ + + +/*[clinic input] +Test.__init__ -> long +Test overriding the __init__ return converter +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test()\n" +"--\n" +"\n" +"Test overriding the __init__ return converter"); + +static long +Test___init___impl(TestObj *self); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + long _return_value; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + _return_value = Test___init___impl((TestObj *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} + +static long +Test___init___impl(TestObj *self) +/*[clinic end generated code: output=daf6ee12c4e443fb input=311af0dc7f17e8e9]*/ + + +/*[clinic input] +fn_with_default_binop_expr + arg: object(c_default='CONST_A + CONST_B') = a+b +[clinic start generated code]*/ + +PyDoc_STRVAR(fn_with_default_binop_expr__doc__, +"fn_with_default_binop_expr($module, /, arg=a+b)\n" +"--\n" +"\n"); + +#define FN_WITH_DEFAULT_BINOP_EXPR_METHODDEF \ + {"fn_with_default_binop_expr", _PyCFunction_CAST(fn_with_default_binop_expr), METH_FASTCALL|METH_KEYWORDS, fn_with_default_binop_expr__doc__}, + +static PyObject * +fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg); + +static PyObject * +fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fn_with_default_binop_expr", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *arg = CONST_A + CONST_B; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + arg = args[0]; +skip_optional_pos: + return_value = fn_with_default_binop_expr_impl(module, arg); + +exit: + return return_value; +} + +static PyObject * +fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ + +/*[python input] +class Custom_converter(CConverter): + type = "str" + default = "Hello!" + converter = "c_converter_func" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d612708f0efb8e3c]*/ + +/*[clinic input] +docstr_fallback_to_converter_default + a: Custom +Check docstring default value fallback. + +Verify that the docstring formatter fetches the default +value from the converter if no 'py_default' is found. +The signature should have the default a='Hello!', +as given by the Custom converter. +[clinic start generated code]*/ + +PyDoc_STRVAR(docstr_fallback_to_converter_default__doc__, +"docstr_fallback_to_converter_default($module, /, a=\'Hello!\')\n" +"--\n" +"\n" +"Check docstring default value fallback.\n" +"\n" +"Verify that the docstring formatter fetches the default\n" +"value from the converter if no \'py_default\' is found.\n" +"The signature should have the default a=\'Hello!\',\n" +"as given by the Custom converter."); + +#define DOCSTR_FALLBACK_TO_CONVERTER_DEFAULT_METHODDEF \ + {"docstr_fallback_to_converter_default", _PyCFunction_CAST(docstr_fallback_to_converter_default), METH_FASTCALL|METH_KEYWORDS, docstr_fallback_to_converter_default__doc__}, + +static PyObject * +docstr_fallback_to_converter_default_impl(PyObject *module, str a); + +static PyObject * +docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "docstr_fallback_to_converter_default", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + str a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!c_converter_func(args[0], &a)) { + goto exit; + } + return_value = docstr_fallback_to_converter_default_impl(module, a); + +exit: + return return_value; +} + +static PyObject * +docstr_fallback_to_converter_default_impl(PyObject *module, str a) +/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 127008d443e4c6..2f945663618722 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -385,6 +385,37 @@ def test_version_directive_unsupported_string(self): """ self.expect_failure(block, err) + def test_clone_mismatch(self): + err = "'kind' of function and cloned function don't match!" + block = """ + /*[clinic input] + module m + @classmethod + m.f1 + a: object + [clinic start generated code]*/ + /*[clinic input] + @staticmethod + m.f2 = m.f1 + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=9) + + def test_badly_formed_return_annotation(self): + err = "Badly formed annotation for m.f: 'Custom'" + block = """ + /*[python input] + class Custom_return_converter(CReturnConverter): + def __init__(self): + raise ValueError("abc") + [python start generated code]*/ + /*[clinic input] + module m + m.f -> Custom + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=8) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): @@ -642,7 +673,7 @@ def test_param_with_continuations(self): p = function.parameters['follow_symlinks'] self.assertEqual(True, p.default) - def test_param_default_expression(self): + def test_param_default_expr_named_constant(self): function = self.parse_function(""" module os os.access @@ -663,6 +694,17 @@ def test_param_default_expression(self): """ self.expect_failure(block, err, lineno=2) + def test_param_default_expr_binop(self): + err = ( + "When you specify an expression ('a + b') as your default value,\n" + "you MUST specify a valid c_default." + ) + block = """ + fn + follow_symlinks: int = a + b + """ + self.expect_failure(block, err, lineno=1) + def test_param_no_docstring(self): function = self.parse_function(""" module os @@ -1241,6 +1283,63 @@ def test_function_not_at_column_0(self): Nested docstring here, goeth. """) + def test_docstring_only_summary(self): + function = self.parse_function(""" + module m + m.f + summary + """) + self.checkDocstring(function, """ + f($module, /) + -- + + summary + """) + + def test_docstring_empty_lines(self): + function = self.parse_function(""" + module m + m.f + + + """) + self.checkDocstring(function, """ + f($module, /) + -- + """) + + def test_docstring_explicit_params_placement(self): + function = self.parse_function(""" + module m + m.f + a: int + Param docstring for 'a' will be included + b: int + c: int + Param docstring for 'c' will be included + This is the summary line. + + We'll now place the params section here: + {parameters} + And now for something completely different! + (Note the added newline) + """) + self.checkDocstring(function, """ + f($module, /, a, b, c) + -- + + This is the summary line. + + We'll now place the params section here: + a + Param docstring for 'a' will be included + c + Param docstring for 'c' will be included + + And now for something completely different! + (Note the added newline) + """) + def test_indent_stack_no_tabs(self): block = """ module foo @@ -1471,7 +1570,100 @@ def test_illegal_c_identifier(self): test.fn a as 17a: int """ - self.expect_failure(block, err) + self.expect_failure(block, err, lineno=2) + + def test_cannot_convert_special_method(self): + err = "__len__ is a special method and cannot be converted" + block = """ + class T "" "" + T.__len__ + """ + self.expect_failure(block, err, lineno=1) + + def test_cannot_specify_pydefault_without_default(self): + err = "You can't specify py_default without specifying a default value!" + block = """ + fn + a: object(py_default='NULL') + """ + self.expect_failure(block, err, lineno=1) + + def test_vararg_cannot_take_default_value(self): + err = "Vararg can't take a default value!" + block = """ + fn + *args: object = None + """ + self.expect_failure(block, err, lineno=1) + + def test_invalid_legacy_converter(self): + err = "fhi is not a valid legacy converter" + block = """ + fn + a: 'fhi' + """ + self.expect_failure(block, err, lineno=1) + + def test_parent_class_or_module_does_not_exist(self): + err = "Parent class or module z does not exist" + block = """ + module m + z.func + """ + self.expect_failure(block, err, lineno=1) + + def test_duplicate_param_name(self): + err = "You can't have two parameters named 'a'" + block = """ + module m + m.func + a: int + a: float + """ + self.expect_failure(block, err, lineno=3) + + def test_param_requires_custom_c_name(self): + err = "Parameter 'module' requires a custom C name" + block = """ + module m + m.func + module: int + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_assert_no_group(self): + err = "Function func has a ] without a matching [." + block = """ + module m + m.func + ] + docstring + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_no_summary(self): + err = "Docstring for m.func does not have a summary line!" + block = """ + module m + m.func + docstring1 + docstring2 + """ + self.expect_failure(block, err, lineno=0) + + def test_state_func_docstring_only_one_param_template(self): + err = "You may not specify {parameters} more than once in a docstring!" + block = """ + module m + m.func + docstring summary + + these are the params: + {parameters} + these are the params again: + {parameters} + """ + self.expect_failure(block, err, lineno=0) class ClinicExternalTest(TestCase): From 77e09192b5f1caf14cd5f92ccb53a4592e83e8bc Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 3 Aug 2023 13:37:14 +0100 Subject: [PATCH 012/598] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (#107586) Co-authored-by: T. Wouters --- .../Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst | 6 ++++++ Modules/_ssl.c | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst new file mode 100644 index 00000000000000..ecaf437a48e0ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst @@ -0,0 +1,6 @@ +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b61d10d9f59fde..c001b875906d44 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -667,6 +667,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { + if (ERR_GET_LIB(e) == ERR_LIB_SSL && + ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { + type = state->PySSLCertVerificationErrorObject; + } p = PY_SSL_ERROR_SYSCALL; } break; From ed4a978449c856372d1a7cd389f91cafe2581c87 Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Thu, 3 Aug 2023 15:19:24 +0100 Subject: [PATCH 013/598] gh-107576: Ensure `__orig_bases__` are our own in `get_original_bases` (#107584) Co-authored-by: Chris Bouchard Co-authored-by: Alex Waygood --- Lib/test/test_types.py | 13 +++++++++++++ Lib/types.py | 11 ++++------- .../2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst | 3 +++ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 81744940f25b82..f2efee90dc0240 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1397,6 +1397,7 @@ class A: pass class B(typing.Generic[T]): pass class C(B[int]): pass class D(B[str], float): pass + self.assertEqual(types.get_original_bases(A), (object,)) self.assertEqual(types.get_original_bases(B), (typing.Generic[T],)) self.assertEqual(types.get_original_bases(C), (B[int],)) @@ -1409,6 +1410,18 @@ class F(list[int]): pass self.assertEqual(types.get_original_bases(E), (list[T],)) self.assertEqual(types.get_original_bases(F), (list[int],)) + class FirstBase(typing.Generic[T]): pass + class SecondBase(typing.Generic[T]): pass + class First(FirstBase[int]): pass + class Second(SecondBase[int]): pass + class G(First, Second): pass + self.assertEqual(types.get_original_bases(G), (First, Second)) + + class First_(typing.Generic[T]): pass + class Second_(typing.Generic[T]): pass + class H(First_, Second_): pass + self.assertEqual(types.get_original_bases(H), (First_, Second_)) + class ClassBasedNamedTuple(typing.NamedTuple): x: int diff --git a/Lib/types.py b/Lib/types.py index 6110e6e1de7249..b4aa19cec40c89 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -165,14 +165,11 @@ class Baz(list[str]): ... assert get_original_bases(int) == (object,) """ try: - return cls.__orig_bases__ + return cls.__dict__.get("__orig_bases__", cls.__bases__) except AttributeError: - try: - return cls.__bases__ - except AttributeError: - raise TypeError( - f'Expected an instance of type, not {type(cls).__name__!r}' - ) from None + raise TypeError( + f"Expected an instance of type, not {type(cls).__name__!r}" + ) from None class DynamicClassAttribute: diff --git a/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst new file mode 100644 index 00000000000000..67677dd3c8ed24 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst @@ -0,0 +1,3 @@ +Fix :func:`types.get_original_bases` to only return +:attr:`!__orig_bases__` if it is present on ``cls`` directly. Patch by +James Hilton-Balfe. From 14fbd4e6b16dcbcbff448b047f7e2faa27bbedba Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 3 Aug 2023 07:04:03 -0800 Subject: [PATCH 014/598] gh-107446: Fix test_inspect.test_class_with_method_from_other_module when ran multiple times (#107451) Co-authored-by: Kumar Aditya --- Lib/test/test_inspect.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 3fbfc073255532..5c31748ce2caac 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -990,6 +990,9 @@ def f(self): with DirsOnSysPath(tempdir): import inspect_actual self.assertIn("correct", inspect.getsource(inspect_actual.A)) + # Remove the module from sys.modules to force it to be reloaded. + # This is necessary when the test is run multiple times. + sys.modules.pop("inspect_actual") @unittest.skipIf( support.is_emscripten or support.is_wasi, From 58ef74186795c56e3ec86e8c8f351a1d7826638a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 3 Aug 2023 13:51:08 -0600 Subject: [PATCH 015/598] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) The linked list of objects was a global variable, which broke isolation between interpreters, causing crashes. To solve this, we've moved the linked list to each interpreter. --- Include/internal/pycore_object.h | 4 +- Include/internal/pycore_object_state.h | 13 +++-- Include/internal/pycore_runtime_init.h | 11 ++++ ...-08-02-12-24-51.gh-issue-107080.PNolFU.rst | 4 ++ Objects/object.c | 51 ++++++++++++------- Python/pylifecycle.c | 8 +-- 6 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b730aba9912830..76b3cd69cbf5a7 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -292,8 +292,8 @@ extern void _PyDebug_PrintTotalRefs(void); #ifdef Py_TRACE_REFS extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(FILE *); -extern void _Py_PrintReferenceAddresses(FILE *); +extern void _Py_PrintReferences(PyInterpreterState *, FILE *); +extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 94005d77881432..65feb5af969f8b 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,17 +11,22 @@ extern "C" { struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#else - int _not_used; #endif + int _not_used; }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#else - int _not_used; #endif +#ifdef Py_TRACE_REFS + /* Head of circular doubly-linked list of all objects. These are linked + * together via the _ob_prev and _ob_next members of a PyObject, which + * exist only in a Py_TRACE_REFS build. + */ + PyObject refchain; +#endif + int _not_used; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index dcfceef4f175a2..e89d368be07aa7 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -157,6 +157,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -186,6 +187,16 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } +#ifdef Py_TRACE_REFS +# define _py_object_state_INIT(INTERP) \ + { \ + .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ + } +#else +# define _py_object_state_INIT(INTERP) \ + { 0 } +#endif + // global objects diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst new file mode 100644 index 00000000000000..5084c854360e35 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst @@ -0,0 +1,4 @@ +Trace refs builds (``--with-trace-refs``) were crashing when used with +isolated subinterpreters. The problematic global state has been isolated to +each interpreter. Other fixing the crashes, this change does not affect +users. diff --git a/Objects/object.c b/Objects/object.c index 8caa5fd0af3cc9..c4413a00ede80c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -159,11 +159,8 @@ _PyDebug_PrintTotalRefs(void) { Do not call them otherwise, they do not initialize the object! */ #ifdef Py_TRACE_REFS -/* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ -static PyObject refchain = {&refchain, &refchain}; + +#define REFCHAIN(interp) &interp->object_state.refchain /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If @@ -188,10 +185,11 @@ _Py_AddToAllObjects(PyObject *op, int force) } #endif if (force || op->_ob_prev == NULL) { - op->_ob_next = refchain._ob_next; - op->_ob_prev = &refchain; - refchain._ob_next->_ob_prev = op; - refchain._ob_next = op; + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + op->_ob_next = refchain->_ob_next; + op->_ob_prev = refchain; + refchain->_ob_next->_ob_prev = op; + refchain->_ob_next = op; } } #endif /* Py_TRACE_REFS */ @@ -2229,7 +2227,8 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - if (op == &refchain || + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + if (op == refchain || op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) { _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); @@ -2237,12 +2236,12 @@ _Py_ForgetReference(PyObject *op) #ifdef SLOW_UNREF_CHECK PyObject *p; - for (p = refchain._ob_next; p != &refchain; p = p->_ob_next) { + for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { if (p == op) { break; } } - if (p == &refchain) { + if (p == refchain) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); @@ -2258,11 +2257,15 @@ _Py_ForgetReference(PyObject *op) * interpreter must be in a healthy state. */ void -_Py_PrintReferences(FILE *fp) +_Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { PyObject *op; + if (interp == NULL) { + interp = _PyInterpreterState_Main(); + } fprintf(fp, "Remaining objects:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) { + PyObject *refchain = REFCHAIN(interp); + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); if (PyObject_Print(op, fp, 0) != 0) { PyErr_Clear(); @@ -2274,34 +2277,42 @@ _Py_PrintReferences(FILE *fp) /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ +// XXX This function is not safe to use if the interpreter has been +// freed or is in an unhealthy state (e.g. late in finalization). +// The call in Py_FinalizeEx() is okay since the main interpreter +// is statically allocated. void -_Py_PrintReferenceAddresses(FILE *fp) +_Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { PyObject *op; + PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) fprintf(fp, "%p [%zd] %s\n", (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); } +/* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { int i, n; PyObject *t = NULL; PyObject *res, *op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyArg_ParseTuple(args, "i|O", &n, &t)) return NULL; - op = refchain._ob_next; + PyObject *refchain = REFCHAIN(interp); + op = refchain->_ob_next; res = PyList_New(0); if (res == NULL) return NULL; - for (i = 0; (n == 0 || i < n) && op != &refchain; i++) { + for (i = 0; (n == 0 || i < n) && op != refchain; i++) { while (op == self || op == args || op == res || op == t || (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { op = op->_ob_next; - if (op == &refchain) + if (op == refchain) return res; } if (PyList_Append(res, op) < 0) { @@ -2313,6 +2324,8 @@ _Py_GetObjects(PyObject *self, PyObject *args) return res; } +#undef REFCHAIN + #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ceca7776a276ed..7a17f92b550c05 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1921,11 +1921,11 @@ Py_FinalizeEx(void) } if (dump_refs) { - _Py_PrintReferences(stderr); + _Py_PrintReferences(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferences(dump_refs_fp); + _Py_PrintReferences(tstate->interp, dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -1961,11 +1961,11 @@ Py_FinalizeEx(void) */ if (dump_refs) { - _Py_PrintReferenceAddresses(stderr); + _Py_PrintReferenceAddresses(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(dump_refs_fp); + _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ From 9e6590b0978876de14587f528a09632b8879c369 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 3 Aug 2023 22:26:14 +0100 Subject: [PATCH 016/598] gh-106368: Argument clinic tests: improve error message when `expect_success()` fails (#107606) --- Lib/test/test_clinic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 2f945663618722..3aa41631d3637a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1680,7 +1680,8 @@ def run_clinic(self, *args): def expect_success(self, *args): out, err, code = self.run_clinic(*args) - self.assertEqual(code, 0, f"Unexpected failure: {args=}") + if code != 0: + self.fail("\n".join([f"Unexpected failure: {args=}", out, err])) self.assertEqual(err, "") return out From ee78d01a61f44c31b8add2bffe687718d2d34d60 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 4 Aug 2023 01:17:17 +0100 Subject: [PATCH 017/598] gh-104146: Argument clinic: remove unused methods and variables (#107608) --- Tools/clinic/clinic.py | 20 +++++--------------- Tools/clinic/cpp.py | 8 -------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1bcdb6b1c3640a..733a83ee58c0f4 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -377,8 +377,10 @@ def flush() -> None: return tuple(version) def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]: - iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) - for i, (a, b) in enumerate(iterator): + iterator = itertools.zip_longest( + version_splitter(version1), version_splitter(version2), fillvalue=0 + ) + for a, b in iterator: if a < b: return -1 if a > b: @@ -4368,19 +4370,11 @@ def depth(self) -> int: """ return len(self.indents) - def indent(self, line: str) -> str: - """ - Indents a line by the currently defined margin. - """ - assert self.margin is not None, "Cannot call .indent() before calling .infer()" - return self.margin + line - def dedent(self, line: str) -> str: """ Dedents a line by the currently defined margin. - (The inverse of 'indent'.) """ - assert self.margin is not None, "Cannot call .indent() before calling .infer()" + assert self.margin is not None, "Cannot call .dedent() before calling .infer()" margin = self.margin indent = self.indents[-1] if not line.startswith(margin): @@ -4641,10 +4635,6 @@ def valid_line(self, line: str) -> bool: return True - @staticmethod - def calculate_indent(line: str) -> int: - return len(line) - len(line.strip()) - def next( self, state: StateKeeper, diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 5b7fa06494ad53..21a1b02e862c1c 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -65,14 +65,6 @@ def fail(self, *a: object) -> NoReturn: print(" ", ' '.join(str(x) for x in a)) sys.exit(-1) - def close(self) -> None: - if self.stack: - self.fail("Ended file while still in a preprocessor conditional block!") - - def write(self, s: str) -> None: - for line in s.split("\n"): - self.writeline(line) - def writeline(self, line: str) -> None: self.line_number += 1 line = line.strip() From e52e87c349feeee77445205829bfc8db9fe4b80e Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 3 Aug 2023 21:37:23 -0700 Subject: [PATCH 018/598] GH-105481: Mark more files as generated (GH-107598) --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index 5d5558da711b17..1f2f8a4cd8735a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -72,9 +72,11 @@ Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated +Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/opcode.h generated Include/token.h generated +Lib/_opcode_metadata.py generated Lib/keyword.py generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated From a443c310ac87f214164cb1e3f8af3f799668c867 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 4 Aug 2023 07:28:25 +0200 Subject: [PATCH 019/598] gh-107609: Fix duplicate module check in Argument Clinic (#107610) Also remove duplicate module def from _testcapi. --- Lib/test/test_clinic.py | 10 ++++++++++ .../2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst | 3 +++ Modules/_testcapi/vectorcall.c | 3 +-- Tools/clinic/clinic.py | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3aa41631d3637a..59669d61f01bd8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -416,6 +416,16 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) + def test_module_already_got_one(self): + err = "Already defined module 'm'!" + block = """ + /*[clinic input] + module m + module m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst new file mode 100644 index 00000000000000..080a6c15d9b8c5 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst @@ -0,0 +1,3 @@ +Fix duplicate module check in Argument Clinic. Previously, a duplicate +definition would incorrectly be silently accepted. Patch by Erlend E. +Aasland. diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 61c6e0f485f47e..2b5110fcba2c91 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -155,10 +155,9 @@ VectorCallClass_vectorcall(PyObject *callable, } /*[clinic input] -module _testcapi class _testcapi.VectorCallClass "PyObject *" "&PyType_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8423a8e919f2f0df]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=95c63c1a47f9a995]*/ /*[clinic input] _testcapi.VectorCallClass.set_vectorcall diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 733a83ee58c0f4..7525c1ca524003 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4472,7 +4472,7 @@ def directive_module(self, name: str) -> None: if cls: fail("Can't nest a module inside a class!") - if name in module.classes: + if name in module.modules: fail("Already defined module " + repr(name) + "!") m = Module(name, module) From 0bd784b355edf0d1911a7830db5d41c84171e367 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 4 Aug 2023 09:48:32 +0200 Subject: [PATCH 020/598] gh-106368: Increase Argument Clinic test coverage (#107611) Add tests for directives and destinations --- Lib/test/test_clinic.py | 296 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 59669d61f01bd8..562c66a37e3294 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -117,6 +117,7 @@ def __repr__(self): class ClinicWholeFileTest(TestCase): + maxDiff = None def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): _expect_failure(self, self.clinic.parse, raw, errmsg, @@ -426,6 +427,230 @@ def test_module_already_got_one(self): """ self.expect_failure(block, err, lineno=3) + def test_destination_already_got_one(self): + err = "Destination already exists: 'test'" + block = """ + /*[clinic input] + destination test new buffer + destination test new buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_does_not_exist(self): + err = "Destination does not exist: '/dev/null'" + block = """ + /*[clinic input] + output everything /dev/null + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_class_already_got_one(self): + err = "Already defined class 'C'!" + block = """ + /*[clinic input] + class C "" "" + class C "" "" + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_cant_nest_module_inside_class(self): + err = "Can't nest a module inside a class!" + block = """ + /*[clinic input] + class C "" "" + module C.m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_dest_buffer_not_empty_at_eof(self): + expected_warning = ("Destination buffer 'buffer' not empty at " + "end of file, emptying.") + expected_generated = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=1c4668687f5fd002]*/ + + /*[clinic input] + dump buffer + [clinic start generated code]*/ + + PyDoc_VAR(fn__doc__); + + PyDoc_STRVAR(fn__doc__, + "fn($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FN_METHODDEF \\ + {"fn", (PyCFunction)fn, METH_O, fn__doc__}, + + static PyObject * + fn(PyObject *module, PyObject *a) + /*[clinic end generated code: output=be6798b148ab4e53 input=524ce2e021e4eba6]*/ + """) + block = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + """) + with support.captured_stdout() as stdout: + generated = self.clinic.parse(block) + self.assertIn(expected_warning, stdout.getvalue()) + self.assertEqual(generated, expected_generated) + + def test_directive_set_misuse(self): + err = "unknown variable 'ets'" + block = """ + /*[clinic input] + set ets tse + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_prefix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '// ' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "// PyDoc_VAR(fn__doc__);" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_suffix(self): + block = dedent(""" + /*[clinic input] + set line_suffix ' // test' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "PyDoc_VAR(fn__doc__); // test" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_prefix_and_suffix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '{block comment start} ' + set line_suffix ' {block comment end}' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "/* PyDoc_VAR(fn__doc__); */" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_printout(self): + block = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + """) + expected = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + test + /*[clinic end generated code: output=4e1243bd22c66e76 input=898f1a32965d44ca]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, expected) + + def test_directive_preserve_twice(self): + err = "Can't have preserve twice in one block!" + block = """ + /*[clinic input] + preserve + preserve + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_directive_preserve_input(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = """ + /*[clinic input] + preserve + fn + a: object + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_directive_preserve_output(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = dedent(""" + /*[clinic input] + output everything buffer + preserve + [clinic start generated code]*/ + // Preserve this + /*[clinic end generated code: output=eaa49677ae4c1f7d input=559b5db18fddae6a]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, block) + + def test_directive_output_invalid_command(self): + err = ( + "Invalid command / destination name 'cmd', must be one of:\n" + " preset push pop print everything cpp_if docstring_prototype " + "docstring_definition methoddef_define impl_prototype " + "parser_prototype parser_definition cpp_endif methoddef_ifndef " + "impl_definition" + ) + block = """ + /*[clinic input] + output cmd buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): @@ -1496,6 +1721,16 @@ class Foo "" "" """ self.expect_failure(block, err, lineno=3) + def test_duplicate_coexist(self): + err = "Called @coexist twice" + block = """ + module m + @coexist + @coexist + m.fn + """ + self.expect_failure(block, err, lineno=2) + def test_unused_param(self): block = self.parse(""" module foo @@ -1931,6 +2166,67 @@ def test_cli_fail_make_without_srcdir(self): msg = "error: --srcdir must not be empty with --make" self.assertIn(msg, err) + def test_file_dest(self): + block = dedent(""" + /*[clinic input] + destination test new file {path}.h + output everything test + func + a: object + / + [clinic start generated code]*/ + """) + expected_checksum_line = ( + "/*[clinic end generated code: " + "output=da39a3ee5e6b4b0d input=b602ab8e173ac3bd]*/\n" + ) + expected_output = dedent("""\ + /*[clinic input] + preserve + [clinic start generated code]*/ + + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # include "pycore_gc.h" // PyGC_Head + # include "pycore_runtime.h" // _Py_ID() + #endif + + + PyDoc_VAR(func__doc__); + + PyDoc_STRVAR(func__doc__, + "func($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FUNC_METHODDEF \\ + {"func", (PyCFunction)func, METH_O, func__doc__}, + + static PyObject * + func(PyObject *module, PyObject *a) + /*[clinic end generated code: output=56c09670e89a0d9a input=a9049054013a1b77]*/ + """) + with os_helper.temp_dir() as tmp_dir: + in_fn = os.path.join(tmp_dir, "test.c") + out_fn = os.path.join(tmp_dir, "test.c.h") + with open(in_fn, "w", encoding="utf-8") as f: + f.write(block) + with open(out_fn, "w", encoding="utf-8") as f: + f.write("") # Write an empty output file! + # Clinic should complain about the empty output file. + _, err = self.expect_failure(in_fn) + expected_err = (f"Modified destination file {out_fn!r}, " + "not overwriting!") + self.assertIn(expected_err, err) + # Run clinic again, this time with the -f option. + out = self.expect_success("-f", in_fn) + # Read back the generated output. + with open(in_fn, encoding="utf-8") as f: + data = f.read() + expected_block = f"{block}{expected_checksum_line}" + self.assertEqual(data, expected_block) + with open(out_fn, encoding="utf-8") as f: + data = f.read() + self.assertEqual(data, expected_output) try: import _testclinic as ac_tester From fa45958450aa3489607daf9855ca0474a2a20878 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 4 Aug 2023 10:10:29 +0100 Subject: [PATCH 021/598] GH-107263: Increase C stack limit for most functions, except `_PyEval_EvalFrameDefault()` (GH-107535) * Set C recursion limit to 1500, set cost of eval loop to 2 frames, and compiler mutliply to 2. --- Doc/whatsnew/3.12.rst | 5 +++++ Include/cpython/pystate.h | 3 ++- Lib/test/list_tests.py | 4 ++-- Lib/test/mapping_tests.py | 3 ++- Lib/test/support/__init__.py | 10 ++++++++- Lib/test/test_ast.py | 1 + Lib/test/test_call.py | 3 ++- Lib/test/test_compile.py | 21 +++++++------------ Lib/test/test_dict.py | 4 ++-- Lib/test/test_dictviews.py | 3 ++- Lib/test/test_exception_group.py | 4 ++-- Lib/test/test_zlib.py | 7 ++----- ...-07-30-05-20-16.gh-issue-107263.q0IU2M.rst | 3 +++ Parser/asdl_c.py | 2 +- Python/Python-ast.c | 2 +- Python/ast.c | 2 +- Python/ast_opt.c | 2 +- Python/bytecodes.c | 2 +- Python/ceval.c | 8 ++++++- Python/generated_cases.c.h | 2 +- Python/symtable.c | 11 ++-------- 21 files changed, 57 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3c39476ced9f48..6553013552d003 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -801,6 +801,11 @@ sys exception instance, rather than to a ``(typ, exc, tb)`` tuple. (Contributed by Irit Katriel in :gh:`103176`.) +* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`. + The recursion limit now applies only to Python code. Builtin functions do + not use the recursion limit, but are protected by a different mechanism + that prevents recursion from causing a virtual machine crash. + tempfile -------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 30de4ee4b6c58c..56e473cc5e42d5 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -222,7 +222,8 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 800 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 1500 # endif #endif diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461f..b1ef332522d2ce 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 613206a0855aea..5492bbf86d1f87 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,6 +2,7 @@ import unittest import collections import sys +from test.support import C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2a59911e54d6b9..64c66d8e25d9cd 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -64,7 +64,8 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "skip_on_s390x", ] @@ -2460,3 +2461,10 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 + +# The default C recursion limit (from Include/cpython/pystate.h). +C_RECURSION_LIMIT = 1500 + +#Windows doesn't have os.uname() but it doesn't support s390x. +skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', + 'skipped on s390x') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a03fa4c7187b05..5346b39043f0f5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1084,6 +1084,7 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 09a531f8cc627b..c3c3b1853b5736 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api +from test.support import cpython_only, requires_limited_api, skip_on_s390x try: import _testcapi except ImportError: @@ -918,6 +918,7 @@ def test_multiple_values(self): @cpython_only class TestRecursion(unittest.TestCase): + @skip_on_s390x def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 85ce0a4b39d854..770184c5ef9a91 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,10 +11,9 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization) + requires_specialization, C_RECURSION_LIMIT) from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -112,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 2000 + repeat = int(C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Expected limit is C_RECURSION_LIMIT * 2 + # Duplicating the limit here is a little ugly. + # Perhaps it should be exposed somewhere... + fail_depth = C_RECURSION_LIMIT * 2 + 1 + crash_depth = C_RECURSION_LIMIT * 100 + success_depth = int(C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 79638340059f65..fbc6ce8282de3c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper +from test.support import import_helper, C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 924f4a6829e19c..2bd9d6eef8cfc6 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,6 +3,7 @@ import pickle import sys import unittest +from test.support import C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -279,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2658e027ff3e2b..a02d54da35e948 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest - +from test.support import C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3dac70eb12852c..2113757254c0ed 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -7,7 +7,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G, skip_on_s390x zlib = import_helper.import_module('zlib') @@ -44,10 +44,7 @@ # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # # Make the assumption that s390x always has an accelerator to simplify the skip -# condition. Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') - +# condition. class VersionTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst new file mode 100644 index 00000000000000..fb0940b456dae5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst @@ -0,0 +1,3 @@ +Increase C recursion limit for functions other than the main interpreter +from 800 to 1500. This should allow functions like ``list.__repr__`` and +``json.dumps`` to handle all the inputs that they could prior to 3.12 diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f159b573ce4727..2a36610527f898 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 412de79397477b..8047b1259c5d86 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13073,7 +13073,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/ast.c b/Python/ast.c index 68600ce683b974..74c97f948d15e6 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Validate(mod_ty mod) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ad1e312b084b74..82e7559e5b629a 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0bea7b59597a26..90e26d3c86b380 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -741,7 +741,7 @@ dummy_func( tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/ceval.c b/Python/ceval.c index 369b9a69152a5c..b85e9677747afb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -624,6 +624,11 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup; # pragma warning(disable:4102) #endif + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -676,6 +681,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->previous = &entry_frame; cframe.current_frame = frame; + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { tstate->c_recursion_remaining--; tstate->py_recursion_remaining--; @@ -907,7 +913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e43b3dedf14f5d..9fa549a1203245 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -903,7 +903,7 @@ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/symtable.c b/Python/symtable.c index 04be3192d6c7c4..e9adbd5d29b1f9 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -282,17 +282,10 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when +/* Using a scaling factor means this should automatically adjust when the recursion limit is adjusted for small or large C stack allocations. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) From 2ba7c7f7b151ff56cf12bf3cab286981bb646c90 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 4 Aug 2023 10:34:23 +0100 Subject: [PATCH 022/598] Add some GC stats to Py_STATS (GH-107581) --- Include/internal/pycore_code.h | 2 ++ Include/pystats.h | 9 +++++++++ Modules/gcmodule.c | 18 +++++++++++++++++ Python/specialize.c | 18 ++++++++++++++++- Tools/scripts/summarize_stats.py | 34 ++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index ee1b85187cbab6..00099376635e9b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -274,6 +274,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0) +#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -287,6 +288,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) +#define GC_STAT_ADD(gen, name, n) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/pystats.h b/Include/pystats.h index 54c9b8d8b3538f..e24aef5fe8072b 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -74,12 +74,21 @@ typedef struct _object_stats { uint64_t optimization_traces_created; uint64_t optimization_traces_executed; uint64_t optimization_uops_executed; + /* Temporary value used during GC */ + uint64_t object_visits; } ObjectStats; +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; + GCStats *gc_stats; } PyStats; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 246c0a9e160aa9..35a35091bf4511 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -460,6 +460,7 @@ update_refs(PyGC_Head *containers) static int visit_decref(PyObject *op, void *parent) { + OBJECT_STAT_INC(object_visits); _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); if (_PyObject_IS_GC(op)) { @@ -498,6 +499,7 @@ subtract_refs(PyGC_Head *containers) static int visit_reachable(PyObject *op, PyGC_Head *reachable) { + OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; } @@ -725,6 +727,7 @@ clear_unreachable_mask(PyGC_Head *unreachable) static int visit_move(PyObject *op, PyGC_Head *tolist) { + OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (gc_is_collecting(gc)) { @@ -1195,6 +1198,12 @@ gc_collect_main(PyThreadState *tstate, int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, int nofail) { + GC_STAT_ADD(generation, collections, 1); +#ifdef Py_STATS + if (_py_stats) { + _py_stats->object_stats.object_visits = 0; + } +#endif int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ @@ -1351,6 +1360,15 @@ gc_collect_main(PyThreadState *tstate, int generation, stats->collected += m; stats->uncollectable += n; + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_py_stats) { + GC_STAT_ADD(generation, object_visits, + _py_stats->object_stats.object_visits); + _py_stats->object_stats.object_visits = 0; + } +#endif + if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(n + m); } diff --git a/Python/specialize.c b/Python/specialize.c index 1669ce17fc804e..de329ef1195cbf 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -18,7 +18,8 @@ */ #ifdef Py_STATS -PyStats _py_stats_struct = { 0 }; +GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; +PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] }; PyStats *_py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ @@ -202,17 +203,32 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->optimization_uops_executed); } +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + } +} + static void print_stats(FILE *out, PyStats *stats) { print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); } void _Py_StatsClear(void) { + for (int i = 0; i < NUM_GENERATIONS; i++) { + _py_gc_stats[i] = (GCStats) { 0 }; + } _py_stats_struct = (PyStats) { 0 }; + _py_stats_struct.gc_stats = _py_gc_stats; } void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9c881897c2de1d..f798b2f772d08a 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -494,6 +494,22 @@ def calculate_object_stats(stats): rows.append((label, value, ratio)) return rows +def calculate_gc_stats(stats): + gc_stats = [] + for key, value in stats.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return [ + (i, gen["collections"], gen["objects collected"], gen["object visits"]) + for (i, gen) in enumerate(gc_stats) + ] + def emit_object_stats(stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): rows = calculate_object_stats(stats) @@ -505,6 +521,22 @@ def emit_comparative_object_stats(base_stats, head_stats): head_rows = calculate_object_stats(head_stats) emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) +def emit_gc_stats(stats): + with Section("GC stats", summary="GC collections and effectiveness"): + rows = calculate_gc_stats(stats) + emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) + +def emit_comparative_gc_stats(base_stats, head_stats): + with Section("GC stats", summary="GC collections and effectiveness"): + base_rows = calculate_gc_stats(base_stats) + head_rows = calculate_gc_stats(head_stats) + emit_table( + ("Generation:", + "Base collections:", "Head collections:", + "Base objects collected:", "Head objects collected:", + "Base object visits:", "Head object visits:"), + join_rows(base_rows, head_rows)) + def get_total(opcode_stats): total = 0 for opcode_stat in opcode_stats: @@ -574,6 +606,7 @@ def output_single_stats(stats): emit_specialization_overview(opcode_stats, total) emit_call_stats(stats) emit_object_stats(stats) + emit_gc_stats(stats) with Section("Meta stats", summary="Meta statistics"): emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])]) @@ -596,6 +629,7 @@ def output_comparative_stats(base_stats, head_stats): ) emit_comparative_call_stats(base_stats, head_stats) emit_comparative_object_stats(base_stats, head_stats) + emit_comparative_gc_stats(base_stats, head_stats) def output_stats(inputs, json_output=None): if len(inputs) == 1: From ac7605ed197e8b2336d44c8ac8aeae6faa90a768 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 4 Aug 2023 14:13:10 +0200 Subject: [PATCH 023/598] gh-107614: Normalise Argument Clinic error messages (#107615) - always wrap the offending line, token, or name in quotes - in most cases, put the entire error message on one line Added tests for uncovered branches that were touched by this PR. Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 117 +++++++++++++++---------- Tools/clinic/clinic.py | 184 +++++++++++++++++++++++----------------- 2 files changed, 179 insertions(+), 122 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 562c66a37e3294..dd17d02519bfa7 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -161,11 +161,8 @@ def test_checksum_mismatch(self): [clinic start generated code]*/ /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ """ - err = ( - 'Checksum mismatch!\n' - 'Expected: 0123456789abcdef\n' - 'Computed: da39a3ee5e6b4b0d\n' - ) + err = ("Checksum mismatch! " + "Expected '0123456789abcdef', computed 'da39a3ee5e6b4b0d'") self.expect_failure(raw, err, filename="test.c", lineno=3) def test_garbage_after_stop_line(self): @@ -403,7 +400,7 @@ def test_clone_mismatch(self): self.expect_failure(block, err, lineno=9) def test_badly_formed_return_annotation(self): - err = "Badly formed annotation for m.f: 'Custom'" + err = "Badly formed annotation for 'm.f': 'Custom'" block = """ /*[python input] class Custom_return_converter(CReturnConverter): @@ -509,6 +506,15 @@ def test_dest_buffer_not_empty_at_eof(self): self.assertIn(expected_warning, stdout.getvalue()) self.assertEqual(generated, expected_generated) + def test_dest_clear(self): + err = "Can't clear destination 'file': it's not of type 'buffer'" + block = """ + /*[clinic input] + destination file clear + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + def test_directive_set_misuse(self): err = "unknown variable 'ets'" block = """ @@ -598,7 +604,7 @@ def test_directive_printout(self): self.assertEqual(generated, expected) def test_directive_preserve_twice(self): - err = "Can't have preserve twice in one block!" + err = "Can't have 'preserve' twice in one block!" block = """ /*[clinic input] preserve @@ -637,13 +643,24 @@ def test_directive_preserve_output(self): self.assertEqual(generated, block) def test_directive_output_invalid_command(self): - err = ( - "Invalid command / destination name 'cmd', must be one of:\n" - " preset push pop print everything cpp_if docstring_prototype " - "docstring_definition methoddef_define impl_prototype " - "parser_prototype parser_definition cpp_endif methoddef_ifndef " - "impl_definition" - ) + err = dedent(""" + Invalid command or destination name 'cmd'. Must be one of: + - 'preset' + - 'push' + - 'pop' + - 'print' + - 'everything' + - 'cpp_if' + - 'docstring_prototype' + - 'docstring_definition' + - 'methoddef_define' + - 'impl_prototype' + - 'parser_prototype' + - 'parser_definition' + - 'cpp_endif' + - 'methoddef_ifndef' + - 'impl_definition' + """).strip() block = """ /*[clinic input] output cmd buffer @@ -919,7 +936,7 @@ def test_param_default_expr_named_constant(self): self.assertEqual("MAXSIZE", p.converter.c_default) err = ( - "When you specify a named constant ('sys.maxsize') as your default value,\n" + "When you specify a named constant ('sys.maxsize') as your default value, " "you MUST specify a valid c_default." ) block = """ @@ -931,7 +948,7 @@ def test_param_default_expr_named_constant(self): def test_param_default_expr_binop(self): err = ( - "When you specify an expression ('a + b') as your default value,\n" + "When you specify an expression ('a + b') as your default value, " "you MUST specify a valid c_default." ) block = """ @@ -954,7 +971,7 @@ def test_param_no_docstring(self): def test_param_default_parameters_out_of_order(self): err = ( - "Can't have a parameter without a default ('something_else')\n" + "Can't have a parameter without a default ('something_else') " "after a parameter with a default!" ) block = """ @@ -1088,7 +1105,7 @@ def test_return_converter_invalid_syntax(self): module os os.stat -> invalid syntax """ - err = "Badly formed annotation for os.stat: 'invalid syntax'" + err = "Badly formed annotation for 'os.stat': 'invalid syntax'" self.expect_failure(block, err) def test_legacy_converter_disallowed_in_return_annotation(self): @@ -1252,8 +1269,8 @@ def test_nested_groups(self): def test_disallowed_grouping__two_top_groups_on_left(self): err = ( - 'Function two_top_groups_on_left has an unsupported group ' - 'configuration. (Unexpected state 2.b)' + "Function 'two_top_groups_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" ) block = """ module foo @@ -1281,7 +1298,7 @@ def test_disallowed_grouping__two_top_groups_on_right(self): ] """ err = ( - "Function two_top_groups_on_right has an unsupported group " + "Function 'two_top_groups_on_right' has an unsupported group " "configuration. (Unexpected state 6.b)" ) self.expect_failure(block, err) @@ -1317,7 +1334,7 @@ def test_disallowed_grouping__group_after_parameter_on_left(self): param: int """ err = ( - "Function group_after_parameter_on_left has an unsupported group " + "Function 'group_after_parameter_on_left' has an unsupported group " "configuration. (Unexpected state 2.b)" ) self.expect_failure(block, err) @@ -1334,7 +1351,7 @@ def test_disallowed_grouping__empty_group_on_left(self): param: int """ err = ( - "Function empty_group has an empty group.\n" + "Function 'empty_group' has an empty group. " "All groups must contain at least one parameter." ) self.expect_failure(block, err) @@ -1351,7 +1368,7 @@ def test_disallowed_grouping__empty_group_on_right(self): ] """ err = ( - "Function empty_group has an empty group.\n" + "Function 'empty_group' has an empty group. " "All groups must contain at least one parameter." ) self.expect_failure(block, err) @@ -1365,7 +1382,7 @@ def test_disallowed_grouping__no_matching_bracket(self): group2: int ] """ - err = "Function empty_group has a ] without a matching [." + err = "Function 'empty_group' has a ']' without a matching '['" self.expect_failure(block, err) def test_no_parameters(self): @@ -1400,7 +1417,7 @@ def test_illegal_module_line(self): foo.bar => int / """ - err = "Illegal function name: foo.bar => int" + err = "Illegal function name: 'foo.bar => int'" self.expect_failure(block, err) def test_illegal_c_basename(self): @@ -1409,7 +1426,7 @@ def test_illegal_c_basename(self): foo.bar as 935 / """ - err = "Illegal C basename: 935" + err = "Illegal C basename: '935'" self.expect_failure(block, err) def test_single_star(self): @@ -1419,7 +1436,7 @@ def test_single_star(self): * * """ - err = "Function bar uses '*' more than once." + err = "Function 'bar' uses '*' more than once." self.expect_failure(block, err) def test_parameters_required_after_star(self): @@ -1429,7 +1446,7 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - err = "Function bar specifies '*' without any parameters afterwards." + err = "Function 'bar' specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): self.expect_failure(block, err) @@ -1442,7 +1459,7 @@ def test_single_slash(self): / """ err = ( - "Function bar has an unsupported group configuration. " + "Function 'bar' has an unsupported group configuration. " "(Unexpected state 0.d)" ) self.expect_failure(block, err) @@ -1456,7 +1473,7 @@ def test_double_slash(self): b: int / """ - err = "Function bar uses '/' more than once." + err = "Function 'bar' uses '/' more than once." self.expect_failure(block, err) def test_mix_star_and_slash(self): @@ -1470,7 +1487,7 @@ def test_mix_star_and_slash(self): / """ err = ( - "Function bar mixes keyword-only and positional-only parameters, " + "Function 'bar' mixes keyword-only and positional-only parameters, " "which is unsupported." ) self.expect_failure(block, err) @@ -1483,7 +1500,7 @@ def test_parameters_not_permitted_after_slash_for_now(self): x: int """ err = ( - "Function bar has an unsupported group configuration. " + "Function 'bar' has an unsupported group configuration. " "(Unexpected state 0.d)" ) self.expect_failure(block, err) @@ -1582,7 +1599,8 @@ def test_indent_stack_no_tabs(self): *vararg1: object \t*vararg2: object """ - err = "Tab characters are illegal in the Clinic DSL." + err = ("Tab characters are illegal in the Clinic DSL: " + r"'\t*vararg2: object'") self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): @@ -1841,8 +1859,17 @@ def test_vararg_cannot_take_default_value(self): """ self.expect_failure(block, err, lineno=1) + def test_default_is_not_of_correct_type(self): + err = ("int_converter: default value 2.5 for field 'a' " + "is not of type 'int'") + block = """ + fn + a: int = 2.5 + """ + self.expect_failure(block, err, lineno=1) + def test_invalid_legacy_converter(self): - err = "fhi is not a valid legacy converter" + err = "'fhi' is not a valid legacy converter" block = """ fn a: 'fhi' @@ -1850,7 +1877,7 @@ def test_invalid_legacy_converter(self): self.expect_failure(block, err, lineno=1) def test_parent_class_or_module_does_not_exist(self): - err = "Parent class or module z does not exist" + err = "Parent class or module 'z' does not exist" block = """ module m z.func @@ -1877,7 +1904,7 @@ def test_param_requires_custom_c_name(self): self.expect_failure(block, err, lineno=2) def test_state_func_docstring_assert_no_group(self): - err = "Function func has a ] without a matching [." + err = "Function 'func' has a ']' without a matching '['" block = """ module m m.func @@ -1887,7 +1914,7 @@ def test_state_func_docstring_assert_no_group(self): self.expect_failure(block, err, lineno=2) def test_state_func_docstring_no_summary(self): - err = "Docstring for m.func does not have a summary line!" + err = "Docstring for 'm.func' does not have a summary line!" block = """ module m m.func @@ -1983,13 +2010,11 @@ def test_cli_force(self): const char *hand_edited = "output block is overwritten"; /*[clinic end generated code: output=bogus input=bogus]*/ """) - fail_msg = dedent(""" - Checksum mismatch! - Expected: bogus - Computed: 2ed19 - Suggested fix: remove all generated code including the end marker, - or use the '-f' option. - """) + fail_msg = ( + "Checksum mismatch! Expected 'bogus', computed '2ed19'. " + "Suggested fix: remove all generated code including the end marker, " + "or use the '-f' option.\n" + ) with os_helper.temp_dir() as tmp_dir: fn = os.path.join(tmp_dir, "test.c") with open(fn, "w", encoding="utf-8") as f: @@ -2214,7 +2239,7 @@ def test_file_dest(self): f.write("") # Write an empty output file! # Clinic should complain about the empty output file. _, err = self.expect_failure(in_fn) - expected_err = (f"Modified destination file {out_fn!r}, " + expected_err = (f"Modified destination file {out_fn!r}; " "not overwriting!") self.assertIn(expected_err, err) # Run clinic again, this time with the -f option. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 7525c1ca524003..ee5e0ef70f20f9 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -290,9 +290,11 @@ def linear_format(s: str, **kwargs: str) -> str: continue if trailing: - fail("Text found after {" + name + "} block marker! It must be on a line by itself.") + fail(f"Text found after {{{name}}} block marker! " + "It must be on a line by itself.") if indent.strip(): - fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.") + fail(f"Non-whitespace characters found before {{{name}}} block marker! " + "It must be on a line by itself.") value = kwargs[name] if not value: @@ -1534,7 +1536,8 @@ def render_function( c.render(p, data) if has_option_groups and (not positional): - fail("You cannot use optional groups ('[' and ']')\nunless all parameters are positional-only ('/').") + fail("You cannot use optional groups ('[' and ']') " + "unless all parameters are positional-only ('/').") # HACK # when we're METH_O, but have a custom return converter, @@ -1871,7 +1874,7 @@ def is_stop_line(line: str) -> bool: for field in shlex.split(arguments): name, equals, value = field.partition('=') if not equals: - fail("Mangled Argument Clinic marker line:", repr(line)) + fail(f"Mangled Argument Clinic marker line: {line!r}") d[name.strip()] = value.strip() if self.verify: @@ -1882,11 +1885,10 @@ def is_stop_line(line: str) -> bool: computed = compute_checksum(output, len(checksum)) if checksum != computed: - fail("Checksum mismatch!\nExpected: {}\nComputed: {}\n" + fail("Checksum mismatch! " + f"Expected {checksum!r}, computed {computed!r}. " "Suggested fix: remove all generated code including " - "the end marker,\n" - "or use the '-f' option." - .format(checksum, computed)) + "the end marker, or use the '-f' option.") else: # put back output output_lines = output.splitlines(keepends=True) @@ -2017,9 +2019,10 @@ def __post_init__(self, args: tuple[str, ...]) -> None: ) extra_arguments = 1 if self.type == "file" else 0 if len(args) < extra_arguments: - fail(f"Not enough arguments for destination {self.name} new {self.type}") + fail(f"Not enough arguments for destination " + f"{self.name!r} new {self.type!r}") if len(args) > extra_arguments: - fail(f"Too many arguments for destination {self.name} new {self.type}") + fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") if self.type =='file': d = {} filename = self.clinic.filename @@ -2041,7 +2044,7 @@ def __repr__(self) -> str: def clear(self) -> None: if self.type != 'buffer': - fail("Can't clear destination" + self.name + " , it's not of type buffer") + fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") self.buffers.clear() def dump(self) -> str: @@ -2220,13 +2223,13 @@ def add_destination( *args: str ) -> None: if name in self.destinations: - fail("Destination already exists: " + repr(name)) + fail(f"Destination already exists: {name!r}") self.destinations[name] = Destination(name, type, self, args) def get_destination(self, name: str) -> Destination: d = self.destinations.get(name) if not d: - fail("Destination does not exist: " + repr(name)) + fail(f"Destination does not exist: {name!r}") return d def get_destination_buffer( @@ -2273,15 +2276,16 @@ def parse(self, input: str) -> str: os.makedirs(dirname) except FileExistsError: if not os.path.isdir(dirname): - fail("Can't write to destination {}, " - "can't make directory {}!".format( - destination.filename, dirname)) + fail(f"Can't write to destination " + f"{destination.filename!r}; " + f"can't make directory {dirname!r}!") if self.verify: with open(destination.filename) as f: parser_2 = BlockParser(f.read(), language=self.language) blocks = list(parser_2) if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): - fail("Modified destination file " + repr(destination.filename) + ", not overwriting!") + fail(f"Modified destination file " + f"{destination.filename!r}; not overwriting!") except FileNotFoundError: pass @@ -2322,7 +2326,8 @@ def _module_and_class( return module, cls child = parent.classes.get(field) if not child: - fail('Parent class or module ' + '.'.join(so_far) + " does not exist.") + fullname = ".".join(so_far) + fail(f"Parent class or module {fullname!r} does not exist.") cls = parent = child return module, cls @@ -2339,12 +2344,12 @@ def parse_file( extension = os.path.splitext(filename)[1][1:] if not extension: - fail("Can't extract file type for file " + repr(filename)) + fail(f"Can't extract file type for file {filename!r}") try: language = extensions[extension](filename) except KeyError: - fail("Can't identify file type for file " + repr(filename)) + fail(f"Can't identify file type for file {filename!r}") with open(filename, encoding="utf-8") as f: raw = f.read() @@ -2823,8 +2828,9 @@ def __init__(self, else: names = [cls.__name__ for cls in self.default_type] types_str = ', '.join(names) - fail("{}: default value {!r} for field {} is not of type {}".format( - self.__class__.__name__, default, name, types_str)) + cls_name = self.__class__.__name__ + fail(f"{cls_name}: default value {default!r} for field " + f"{name!r} is not of type {types_str!r}") self.default = default if c_default: @@ -3146,7 +3152,7 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: if accept == {int}: self.format_unit = 'i' elif accept != {object}: - fail("bool_converter: illegal 'accept' argument " + repr(accept)) + fail(f"bool_converter: illegal 'accept' argument {accept!r}") if self.default is not unspecified: self.default = bool(self.default) self.c_default = str(int(self.default)) @@ -3196,7 +3202,7 @@ class char_converter(CConverter): def converter_init(self) -> None: if isinstance(self.default, self.default_type): if len(self.default) != 1: - fail("char_converter: illegal default value " + repr(self.default)) + fail(f"char_converter: illegal default value {self.default!r}") self.c_default = repr(bytes(self.default))[1:] if self.c_default == '"\'"': @@ -3335,7 +3341,7 @@ def converter_init( if accept == {str}: self.format_unit = 'C' elif accept != {int}: - fail("int_converter: illegal 'accept' argument " + repr(accept)) + fail(f"int_converter: illegal 'accept' argument {accept!r}") if type is not None: self.type = type @@ -3472,7 +3478,7 @@ def converter_init(self, *, accept: TypeSet = {int}) -> None: elif accept == {int, NoneType}: self.converter = '_Py_convert_optional_to_ssize_t' else: - fail("Py_ssize_t_converter: illegal 'accept' argument " + repr(accept)) + fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'n': @@ -3502,7 +3508,7 @@ def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: elif accept == {int, NoneType}: self.converter = '_PyEval_SliceIndex' else: - fail("slice_index_converter: illegal 'accept' argument " + repr(accept)) + fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") class size_t_converter(CConverter): type = 'size_t' @@ -3850,7 +3856,7 @@ def converter_init( elif accept == {str, NoneType}: self.converter = '_PyUnicode_WideCharString_Opt_Converter' else: - fail("Py_UNICODE_converter: illegal 'accept' argument " + repr(accept)) + fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") self.c_default = "NULL" def cleanup(self) -> str: @@ -4378,7 +4384,7 @@ def dedent(self, line: str) -> str: margin = self.margin indent = self.indents[-1] if not line.startswith(margin): - fail('Cannot dedent, line does not start with the previous margin:') + fail('Cannot dedent; line does not start with the previous margin.') return line[indent:] @@ -4464,7 +4470,9 @@ def reset(self) -> None: def directive_version(self, required: str) -> None: global version if version_comparitor(version, required) < 0: - fail("Insufficient Clinic version!\n Version: " + version + "\n Required: " + required) + fail("Insufficient Clinic version!\n" + f" Version: {version}\n" + f" Required: {required}") def directive_module(self, name: str) -> None: fields = name.split('.')[:-1] @@ -4473,7 +4481,7 @@ def directive_module(self, name: str) -> None: fail("Can't nest a module inside a class!") if name in module.modules: - fail("Already defined module " + repr(name) + "!") + fail(f"Already defined module {name!r}!") m = Module(name, module) module.modules[name] = m @@ -4491,7 +4499,7 @@ def directive_class( parent = cls or module if name in parent.classes: - fail("Already defined class " + repr(name) + "!") + fail(f"Already defined class {name!r}!") c = Class(name, module, cls, typedef, type_object) parent.classes[name] = c @@ -4499,7 +4507,7 @@ def directive_class( def directive_set(self, name: str, value: str) -> None: if name not in ("line_prefix", "line_suffix"): - fail("unknown variable", repr(name)) + fail(f"unknown variable {name!r}") value = value.format_map({ 'block comment start': '/*', @@ -4520,7 +4528,7 @@ def directive_destination( case "clear": self.clinic.get_destination(name).clear() case _: - fail("unknown destination command", repr(command)) + fail(f"unknown destination command {command!r}") def directive_output( @@ -4533,7 +4541,7 @@ def directive_output( if command_or_name == "preset": preset = self.clinic.presets.get(destination) if not preset: - fail("Unknown preset " + repr(destination) + "!") + fail(f"Unknown preset {destination!r}!") fd.update(preset) return @@ -4562,7 +4570,11 @@ def directive_output( return if command_or_name not in fd: - fail("Invalid command / destination name " + repr(command_or_name) + ", must be one of:\n preset push pop print everything " + " ".join(fd)) + allowed = ["preset", "push", "pop", "print", "everything"] + allowed.extend(fd) + fail(f"Invalid command or destination name {command_or_name!r}. " + "Must be one of:\n -", + "\n - ".join([repr(word) for word in allowed])) fd[command_or_name] = d def directive_dump(self, name: str) -> None: @@ -4574,7 +4586,7 @@ def directive_printout(self, *args: str) -> None: def directive_preserve(self) -> None: if self.preserve_output: - fail("Can't have preserve twice in one block!") + fail("Can't have 'preserve' twice in one block!") self.preserve_output = True def at_classmethod(self) -> None: @@ -4601,7 +4613,8 @@ def parse(self, block: Block) -> None: lines = block.input.split('\n') for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): if '\t' in line: - fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) + fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', + line_number=block_start) try: self.state(line) except ClinicError as exc: @@ -4712,7 +4725,8 @@ def state_modulename_name(self, line: str) -> None: module, cls = self.clinic._module_and_class(fields) if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): - fail("'kind' of function and cloned function don't match! (@classmethod/@staticmethod/@coexist)") + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") function = existing_function.copy( name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, docstring='' @@ -4731,9 +4745,9 @@ def state_modulename_name(self, line: str) -> None: c_basename = c_basename.strip() or None if not is_legal_py_identifier(full_name): - fail("Illegal function name:", full_name) + fail(f"Illegal function name: {full_name!r}") if c_basename and not is_legal_c_identifier(c_basename): - fail("Illegal C basename:", c_basename) + fail(f"Illegal C basename: {c_basename!r}") return_converter = None if returns: @@ -4741,7 +4755,7 @@ def state_modulename_name(self, line: str) -> None: try: module_node = ast.parse(ast_input) except SyntaxError: - fail(f"Badly formed annotation for {full_name}: {returns!r}") + fail(f"Badly formed annotation for {full_name!r}: {returns!r}") function_node = module_node.body[0] assert isinstance(function_node, ast.FunctionDef) try: @@ -4752,7 +4766,7 @@ def state_modulename_name(self, line: str) -> None: fail(f"No available return converter called {name!r}") return_converter = return_converters[name](**kwargs) except ValueError: - fail(f"Badly formed annotation for {full_name}: {returns!r}") + fail(f"Badly formed annotation for {full_name!r}: {returns!r}") fields = [x.strip() for x in full_name.split('.')] function_name = fields.pop() @@ -4777,7 +4791,7 @@ def state_modulename_name(self, line: str) -> None: return_converter = CReturnConverter() if not module: - fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".") + fail("Undefined module used in declaration of {full_name.strip()!r}.") self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, return_converter=return_converter, kind=self.kind, coexist=self.coexist) self.block.signatures.append(self.function) @@ -4963,18 +4977,22 @@ def parse_parameter(self, line: str) -> None: except SyntaxError: pass if not module: - fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) + fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", + repr(line)) function = module.body[0] assert isinstance(function, ast.FunctionDef) function_args = function.args if len(function_args.args) > 1: - fail("Function " + self.function.name + " has an invalid parameter declaration (comma?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (comma?): {line!r}") if function_args.defaults or function_args.kw_defaults: - fail("Function " + self.function.name + " has an invalid parameter declaration (default value?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (default value?): {line!r}") if function_args.kwarg: - fail("Function " + self.function.name + " has an invalid parameter declaration (**kwargs?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") if function_args.vararg: is_vararg = True @@ -4988,7 +5006,7 @@ def parse_parameter(self, line: str) -> None: if not default: if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r})\n" + fail(f"Can't have a parameter without a default ({parameter_name!r}) " "after a parameter with a default!") value: Sentinels | Null if is_vararg: @@ -5048,10 +5066,10 @@ def bad_node(self, node: ast.AST) -> None: except NameError: pass # probably a named constant except Exception as e: - fail("Malformed expression given as default value\n" - "{!r} caused {!r}".format(default, e)) + fail("Malformed expression given as default value " + f"{default!r} caused {e!r}") if bad: - fail("Unsupported expression as default value: " + repr(default)) + fail(f"Unsupported expression as default value: {default!r}") assignment = module.body[0] assert isinstance(assignment, ast.Assign) @@ -5069,7 +5087,10 @@ def bad_node(self, node: ast.AST) -> None: )): c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): - fail("When you specify an expression (" + repr(default) + ") as your default value,\nyou MUST specify a valid c_default." + ast.dump(expr)) + fail(f"When you specify an expression ({default!r}) " + f"as your default value, " + f"you MUST specify a valid c_default.", + ast.dump(expr)) py_default = default value = unknown elif isinstance(expr, ast.Attribute): @@ -5079,13 +5100,16 @@ def bad_node(self, node: ast.AST) -> None: a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - fail("Unsupported default value " + repr(default) + " (looked like a Python constant)") + fail(f"Unsupported default value {default!r} " + "(looked like a Python constant)") a.append(n.id) py_default = ".".join(reversed(a)) c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + fail(f"When you specify a named constant ({py_default!r}) " + "as your default value, " + "you MUST specify a valid c_default.") try: value = eval(py_default) @@ -5102,13 +5126,15 @@ def bad_node(self, node: ast.AST) -> None: c_default = py_default except SyntaxError as e: - fail("Syntax error: " + repr(e.text)) + fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown c_default = kwargs.get("c_default") py_default = default if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + fail("When you specify a named constant " + f"({py_default!r}) as your default value, " + "you MUST specify a valid c_default.") kwargs.setdefault('c_default', c_default) kwargs.setdefault('py_default', py_default) @@ -5116,7 +5142,7 @@ def bad_node(self, node: ast.AST) -> None: dict = legacy_converters if legacy else converters legacy_str = "legacy " if legacy else "" if name not in dict: - fail(f'{name} is not a valid {legacy_str}converter') + fail(f'{name!r} is not a valid {legacy_str}converter') # if you use a c_name for the parameter, we just give that name to the converter # but the parameter object gets the python name converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) @@ -5141,7 +5167,8 @@ def bad_node(self, node: ast.AST) -> None: self.parameter_state = ParamState.START self.function.parameters.clear() else: - fail("A 'self' parameter, if specified, must be the very first thing in the parameter block.") + fail("A 'self' parameter, if specified, must be the " + "very first thing in the parameter block.") if isinstance(converter, defining_class_converter): _lp = len(self.function.parameters) @@ -5153,16 +5180,18 @@ def bad_node(self, node: ast.AST) -> None: if self.group: fail("A 'defining_class' parameter cannot be in an optional group.") else: - fail("A 'defining_class' parameter, if specified, must either be the first thing in the parameter block, or come just after 'self'.") + fail("A 'defining_class' parameter, if specified, must either " + "be the first thing in the parameter block, or come just " + "after 'self'.") p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) names = [k.name for k in self.function.parameters.values()] if parameter_name in names[1:]: - fail("You can't have two parameters named " + repr(parameter_name) + "!") + fail(f"You can't have two parameters named {parameter_name!r}!") elif names and parameter_name == names[0] and c_name is None: - fail(f"Parameter '{parameter_name}' requires a custom C name") + fail(f"Parameter {parameter_name!r} requires a custom C name") key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p @@ -5192,7 +5221,7 @@ def parse_converter( def parse_star(self, function: Function) -> None: """Parse keyword-only parameter marker '*'.""" if self.keyword_only: - fail(f"Function {function.name} uses '*' more than once.") + fail(f"Function {function.name!r} uses '*' more than once.") self.keyword_only = True def parse_opening_square_bracket(self, function: Function) -> None: @@ -5203,7 +5232,8 @@ def parse_opening_square_bracket(self, function: Function) -> None: case ParamState.REQUIRED | ParamState.GROUP_AFTER: self.parameter_state = ParamState.GROUP_AFTER case st: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " f"(Unexpected state {st}.b)") self.group += 1 function.docstring_only = True @@ -5211,9 +5241,9 @@ def parse_opening_square_bracket(self, function: Function) -> None: def parse_closing_square_bracket(self, function: Function) -> None: """Parse closing parameter group symbol ']'.""" if not self.group: - fail(f"Function {function.name} has a ] without a matching [.") + fail(f"Function {function.name!r} has a ']' without a matching '['.") if not any(p.group == self.group for p in function.parameters.values()): - fail(f"Function {function.name} has an empty group.\n" + fail(f"Function {function.name!r} has an empty group. " "All groups must contain at least one parameter.") self.group -= 1 match self.parameter_state: @@ -5222,13 +5252,14 @@ def parse_closing_square_bracket(self, function: Function) -> None: case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: self.parameter_state = ParamState.RIGHT_SQUARE_AFTER case st: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " f"(Unexpected state {st}.c)") def parse_slash(self, function: Function) -> None: """Parse positional-only parameter marker '/'.""" if self.positional_only: - fail(f"Function {function.name} uses '/' more than once.") + fail(f"Function {function.name!r} uses '/' more than once.") self.positional_only = True # REQUIRED and OPTIONAL are allowed here, that allows positional-only # without option groups to work (and have default values!) @@ -5239,10 +5270,10 @@ def parse_slash(self, function: Function) -> None: ParamState.GROUP_BEFORE, } if (self.parameter_state not in allowed) or self.group: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} has an unsupported group configuration. " f"(Unexpected state {self.parameter_state}.d)") if self.keyword_only: - fail(f"Function {function.name} mixes keyword-only and " + fail(f"Function {function.name!r} mixes keyword-only and " "positional-only parameters, which is unsupported.") # fixup preceding parameters for p in function.parameters.values(): @@ -5251,7 +5282,7 @@ def parse_slash(self, function: Function) -> None: if (p.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD and not isinstance(p.converter, self_converter) ): - fail(f"Function {function.name} mixes keyword-only and " + fail(f"Function {function.name!r} mixes keyword-only and " "positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY @@ -5301,7 +5332,7 @@ def state_function_docstring(self, line: str) -> None: assert self.function is not None if self.group: - fail("Function " + self.function.name + " has a ] without a matching [.") + fail(f"Function {self.function.name!r} has a ']' without a matching '['.") if not self.valid_line(line): return @@ -5528,9 +5559,9 @@ def add_parameter(text: str) -> None: if len(lines) >= 2: if lines[1]: - fail("Docstring for " + f.full_name + " does not have a summary line!\n" + - "Every non-blank function docstring must start with\n" + - "a single line summary followed by an empty line.") + fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" + "Every non-blank function docstring must start with " + "a single line summary followed by an empty line.") elif len(lines) == 1: # the docstring is only one line right now--the summary line. # add an empty line after the summary line so we have space @@ -5573,7 +5604,8 @@ def do_post_block_processing_cleanup(self) -> None: last_parameter = next(reversed(list(values))) no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY if no_parameter_after_star: - fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") + fail(f"Function {self.function.name!r} specifies '*' " + "without any parameters afterwards.") self.function.docstring = self.format_docstring() From 7eba274fb3fb959261c6b8fdf0cee10c82004f0e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 4 Aug 2023 14:28:13 +0100 Subject: [PATCH 024/598] gh-104146: Argument clinic: Remove dead code flagged by mypy's `truthy-bool` check (#107627) --- Tools/clinic/clinic.py | 8 ++------ Tools/clinic/mypy.ini | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ee5e0ef70f20f9..6c5a2c7c857c53 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -886,9 +886,7 @@ def output_templates( converters = [p.converter for p in parameters] has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - default_return_converter = (not f.return_converter or - f.return_converter.type == 'PyObject *') - + default_return_converter = f.return_converter.type == 'PyObject *' new_or_init = f.kind.new_or_init vararg: int | str = NO_VARARG @@ -4428,7 +4426,7 @@ class DSLParser: keyword_only: bool positional_only: bool group: int - parameter_state: int + parameter_state: ParamState seen_positional_with_default: bool indent: IndentStack kind: FunctionKind @@ -4790,8 +4788,6 @@ def state_modulename_name(self, line: str) -> None: if not return_converter: return_converter = CReturnConverter() - if not module: - fail("Undefined module used in declaration of {full_name.strip()!r}.") self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, return_converter=return_converter, kind=self.kind, coexist=self.coexist) self.block.signatures.append(self.function) diff --git a/Tools/clinic/mypy.ini b/Tools/clinic/mypy.ini index 4cfc05bec01608..b1fdad673c61a1 100644 --- a/Tools/clinic/mypy.ini +++ b/Tools/clinic/mypy.ini @@ -8,5 +8,5 @@ python_version = 3.10 # and be strict! strict = True strict_concatenate = True -enable_error_code = ignore-without-code,redundant-expr +enable_error_code = ignore-without-code,redundant-expr,truthy-bool warn_unreachable = True From 19f32b24b2e1680ff9929bb64d681397b259c6fb Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 4 Aug 2023 17:16:10 +0300 Subject: [PATCH 025/598] Docs: upgrade to python-docs-theme 2023.7 (#107617) --- Doc/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/requirements.txt b/Doc/requirements.txt index bde509febf5bde..e9b6e56b122895 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,7 +7,7 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # PR #104777: Sphinx 6.2 no longer uses imghdr, removed in Python 3.13. -sphinx==6.2.0 +sphinx==6.2.1 blurb @@ -15,6 +15,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2022.1 +python-docs-theme>=2023.7 -c constraints.txt From 09a8cc79846ce0870e51fa8e7c449e153832fe4b Mon Sep 17 00:00:00 2001 From: Tomas R Date: Fri, 4 Aug 2023 16:34:04 +0200 Subject: [PATCH 026/598] gh-107600: Docs: Update ctypes.ArgumentError error message (#107601) --- Doc/library/ctypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c253a45e1a8b54..ea7873a885172e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -361,7 +361,7 @@ from within *IDLE* or *PythonWin*:: >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 + ctypes.ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 >>> As has been mentioned before, all Python types except integers, strings, and @@ -444,7 +444,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: wrong type + ctypes.ArgumentError: argument 2: TypeError: 'int' object cannot be interpreted as ctypes.c_char_p >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 13 From 407d7fda94268d70b92e2e685ca9891ec3641e78 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 4 Aug 2023 15:56:50 +0100 Subject: [PATCH 027/598] gh-104146: Argument clinic: remove dead code highlighted by the `vulture` tool (#107632) --- Tools/clinic/clinic.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6c5a2c7c857c53..6eb2c550e696fc 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1684,27 +1684,30 @@ class Block: found on the start line of the block between the square brackets. - signatures is either list or None. If it's a list, - it may only contain clinic.Module, clinic.Class, and + signatures is a list. + It may only contain clinic.Module, clinic.Class, and clinic.Function objects. At the moment it should contain at most one of each. output is either str or None. If str, it's the output from this block, with embedded '\n' characters. - indent is either str or None. It's the leading whitespace + indent is a str. It's the leading whitespace that was found on every line of input. (If body_prefix is not empty, this is the indent *after* removing the body_prefix.) - preindent is either str or None. It's the whitespace that + "indent" is different from the concept of "preindent" + (which is not stored as state on Block objects). + "preindent" is the whitespace that was found in front of every line of input *before* the "body_prefix" (see the Language object). If body_prefix is empty, preindent must always be empty too. - To illustrate indent and preindent: Assume that '_' - represents whitespace. If the block processed was in a - Python file, and looked like this: + To illustrate the difference between "indent" and "preindent": + + Assume that '_' represents whitespace. + If the block processed was in a Python file, and looked like this: ____#/*[python] ____#__for a in range(20): ____#____print(a) @@ -1717,7 +1720,6 @@ class Block: signatures: list[Module | Class | Function] = dc.field(default_factory=list) output: Any = None # TODO: Very dynamic; probably untypeable in its current form? indent: str = '' - preindent: str = '' def __repr__(self) -> str: dsl_name = self.dsl_name or "text" @@ -2049,12 +2051,8 @@ def dump(self) -> str: return self.buffers.dump() -# maps strings to Language objects. -# "languages" maps the name of the language ("C", "Python"). -# "extensions" maps the file extension ("c", "py"). +# "extensions" maps the file extension ("c", "py") to Language classes. LangDict = dict[str, Callable[[str], Language]] - -languages = { 'C': CLanguage, 'Python': PythonLanguage } extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } extensions['py'] = PythonLanguage @@ -4427,7 +4425,6 @@ class DSLParser: positional_only: bool group: int parameter_state: ParamState - seen_positional_with_default: bool indent: IndentStack kind: FunctionKind coexist: bool @@ -4458,7 +4455,6 @@ def reset(self) -> None: self.positional_only = False self.group = 0 self.parameter_state: ParamState = ParamState.START - self.seen_positional_with_default = False self.indent = IndentStack() self.kind = CALLABLE self.coexist = False From 400835ea1626c8c6dcd967c7eabe0dad4a923182 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 4 Aug 2023 09:35:56 -0700 Subject: [PATCH 028/598] gh-106812: Refactor cases_generator to allow uops with array stack effects (#107564) Introducing a new file, stacking.py, that takes over several responsibilities related to symbolic evaluation of push/pop operations, with more generality. --- Include/internal/pycore_opcode_metadata.h | 19 +- Lib/test/test_generated_cases.py | 158 +-- Python/executor_cases.c.h | 611 +++++++---- Python/generated_cases.c.h | 1194 ++++++++++++--------- Tools/cases_generator/analysis.py | 171 ++- Tools/cases_generator/flags.py | 4 +- Tools/cases_generator/formatting.py | 51 +- Tools/cases_generator/generate_cases.py | 145 +-- Tools/cases_generator/instructions.py | 109 +- Tools/cases_generator/parsing.py | 21 +- Tools/cases_generator/plexer.py | 13 +- Tools/cases_generator/stacking.py | 400 +++++++ 12 files changed, 1798 insertions(+), 1098 deletions(-) create mode 100644 Tools/cases_generator/stacking.py diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 64a084618e13b8..1cab6c984f3ace 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -679,9 +679,9 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_GLOBAL_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_GLOBAL_BUILTIN: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case DELETE_FAST: return 0; case MAKE_CELL: @@ -739,7 +739,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_METHOD: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_MODULE: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_WITH_HINT: @@ -944,7 +944,18 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC0, INSTR_FMT_IXC00, INSTR_FMT_IXC000 }; +enum InstructionFormat { + INSTR_FMT_IB, + INSTR_FMT_IBC, + INSTR_FMT_IBC00, + INSTR_FMT_IBC000, + INSTR_FMT_IBC00000000, + INSTR_FMT_IX, + INSTR_FMT_IXC, + INSTR_FMT_IXC0, + INSTR_FMT_IXC00, + INSTR_FMT_IXC000, +}; #define IS_VALID_OPCODE(OP) \ (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 9c23c2e82058c7..54378fced54699 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -6,9 +6,9 @@ test_tools.skip_if_missing('cases_generator') with test_tools.imports_under_tool('cases_generator'): + import generate_cases import analysis import formatting - import generate_cases from parsing import StackEffect @@ -46,28 +46,11 @@ def test_effect_sizes(self): (2, "(oparg<<1)"), ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(input_effects), - ), "1 + oparg + oparg*2", - ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(output_effects), - ), - "2 + oparg*4", - ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(other_effects), - ), - "2 + (oparg<<1)", - ) - class TestGeneratedCases(unittest.TestCase): def setUp(self) -> None: super().setUp() + self.maxDiff = None self.temp_dir = tempfile.gettempdir() self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") @@ -140,7 +123,8 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; spam(); STACK_SHRINK(1); DISPATCH(); @@ -173,8 +157,9 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; spam(); stack_pointer[-1] = res; DISPATCH(); @@ -190,9 +175,11 @@ def test_binary_op(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; spam(); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -209,9 +196,11 @@ def test_overlap(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *result; + right = stack_pointer[-1]; + left = stack_pointer[-2]; spam(); stack_pointer[-1] = result; DISPATCH(); @@ -235,8 +224,9 @@ def test_predictions_and_eval_breaker(self): } TARGET(OP3) { - PyObject *arg = stack_pointer[-1]; + PyObject *arg; PyObject *res; + arg = stack_pointer[-1]; DEOPT_IF(xxx, OP1); stack_pointer[-1] = res; CHECK_EVAL_BREAKER(); @@ -281,9 +271,11 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; if (cond) goto pop_2_label; STACK_SHRINK(1); stack_pointer[-1] = res; @@ -299,7 +291,8 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; uint16_t counter = read_u16(&next_instr[0].cache); uint32_t extra = read_u32(&next_instr[1].cache); STACK_SHRINK(1); @@ -338,8 +331,10 @@ def test_macro_instruction(self): """ output = """ TARGET(OP1) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); op1(left, right); next_instr += 1; @@ -347,38 +342,38 @@ def test_macro_instruction(self): } TARGET(OP) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; - PyObject *_tmp_3 = stack_pointer[-3]; + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *arg2; + PyObject *res; + // OP1 + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; uint16_t counter = read_u16(&next_instr[0].cache); op1(left, right); - _tmp_2 = left; - _tmp_1 = right; } + // OP2 + arg2 = stack_pointer[-3]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *arg2 = _tmp_3; - PyObject *res; uint32_t extra = read_u32(&next_instr[3].cache); res = op2(arg2, left, right); - _tmp_3 = res; } - next_instr += 5; - static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); STACK_SHRINK(2); - stack_pointer[-1] = _tmp_3; + stack_pointer[-1] = res; + next_instr += 5; DISPATCH(); } TARGET(OP3) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *arg2 = stack_pointer[-3]; + PyObject *right; + PyObject *left; + PyObject *arg2; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + arg2 = stack_pointer[-3]; res = op3(arg2, left, right); STACK_SHRINK(2); stack_pointer[-1] = res; @@ -396,9 +391,12 @@ def test_array_input(self): """ output = """ TARGET(OP) { - PyObject *above = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg*2)); - PyObject *below = stack_pointer[-(2 + oparg*2)]; + PyObject *above; + PyObject **values; + PyObject *below; + above = stack_pointer[-1]; + values = stack_pointer - 1 - oparg*2; + below = stack_pointer[-2 - oparg*2]; spam(); STACK_SHRINK(oparg*2); STACK_SHRINK(2); @@ -416,12 +414,13 @@ def test_array_output(self): output = """ TARGET(OP) { PyObject *below; - PyObject **values = stack_pointer - (2) + 1; + PyObject **values; PyObject *above; + values = stack_pointer - 1; spam(values, oparg); STACK_GROW(oparg*3); + stack_pointer[-2 - oparg*3] = below; stack_pointer[-1] = above; - stack_pointer[-(2 + oparg*3)] = below; DISPATCH(); } """ @@ -435,8 +434,9 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *above; + values = stack_pointer - oparg; spam(values, oparg); STACK_GROW(1); stack_pointer[-1] = above; @@ -453,8 +453,10 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { - PyObject **values = (stack_pointer - oparg); - PyObject *extra = stack_pointer[-(1 + oparg)]; + PyObject **values; + PyObject *extra; + values = stack_pointer - oparg; + extra = stack_pointer[-1 - oparg]; if (oparg == 0) { STACK_SHRINK(oparg); goto pop_1_somewhere; } STACK_SHRINK(oparg); STACK_SHRINK(1); @@ -471,18 +473,21 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { - PyObject *cc = stack_pointer[-1]; - PyObject *input = ((oparg & 1) == 1) ? stack_pointer[-(1 + (((oparg & 1) == 1) ? 1 : 0))] : NULL; - PyObject *aa = stack_pointer[-(2 + (((oparg & 1) == 1) ? 1 : 0))]; + PyObject *cc; + PyObject *input = NULL; + PyObject *aa; PyObject *xx; PyObject *output = NULL; PyObject *zz; + cc = stack_pointer[-1]; + if ((oparg & 1) == 1) { input = stack_pointer[-1 - ((oparg & 1) == 1 ? 1 : 0)]; } + aa = stack_pointer[-2 - ((oparg & 1) == 1 ? 1 : 0)]; output = spam(oparg, input); STACK_SHRINK((((oparg & 1) == 1) ? 1 : 0)); STACK_GROW(((oparg & 2) ? 1 : 0)); + stack_pointer[-2 - (oparg & 2 ? 1 : 0)] = xx; + if (oparg & 2) { stack_pointer[-1 - (oparg & 2 ? 1 : 0)] = output; } stack_pointer[-1] = zz; - if (oparg & 2) { stack_pointer[-(1 + ((oparg & 2) ? 1 : 0))] = output; } - stack_pointer[-(2 + ((oparg & 2) ? 1 : 0))] = xx; DISPATCH(); } """ @@ -500,29 +505,28 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; - PyObject *_tmp_3 = stack_pointer[-3]; + PyObject *right; + PyObject *middle; + PyObject *left; + PyObject *deep; + PyObject *extra = NULL; + PyObject *res; + // A + right = stack_pointer[-1]; + middle = stack_pointer[-2]; + left = stack_pointer[-3]; { - PyObject *right = _tmp_1; - PyObject *middle = _tmp_2; - PyObject *left = _tmp_3; # Body of A } + // B { - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; # Body of B - _tmp_3 = deep; - if (oparg) { _tmp_2 = extra; } - _tmp_1 = res; } STACK_SHRINK(1); STACK_GROW((oparg ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg) { stack_pointer[-2] = _tmp_2; } - stack_pointer[-3] = _tmp_3; + stack_pointer[-2 - (oparg ? 1 : 0)] = deep; + if (oparg) { stack_pointer[-1 - (oparg ? 1 : 0)] = extra; } + stack_pointer[-1] = res; DISPATCH(); } """ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ed7cf4c15ec4a2..9363b4955087db 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -47,14 +47,16 @@ } case STORE_FAST: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); break; } case POP_TOP: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); break; @@ -69,8 +71,10 @@ } case END_SEND: { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; Py_DECREF(receiver); STACK_SHRINK(1); stack_pointer[-1] = value; @@ -78,8 +82,9 @@ } case UNARY_NEGATIVE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -88,8 +93,9 @@ } case UNARY_NOT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; @@ -98,8 +104,9 @@ case TO_BOOL: { static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -119,15 +126,17 @@ } case TO_BOOL_BOOL: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); break; } case TO_BOOL_INT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { @@ -143,8 +152,9 @@ } case TO_BOOL_LIST: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; @@ -154,8 +164,9 @@ } case TO_BOOL_NONE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -165,8 +176,9 @@ } case TO_BOOL_STR: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { @@ -183,8 +195,9 @@ } case TO_BOOL_ALWAYS_TRUE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; uint32_t version = (uint32_t)operand; // This one is a bit weird, because we expect *some* failures: assert(version); @@ -197,8 +210,9 @@ } case UNARY_INVERT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -207,17 +221,21 @@ } case _GUARD_BOTH_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_MULTIPLY_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -229,9 +247,11 @@ } case _BINARY_OP_ADD_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -243,9 +263,11 @@ } case _BINARY_OP_SUBTRACT_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -257,17 +279,21 @@ } case _GUARD_BOTH_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -279,9 +305,11 @@ } case _BINARY_OP_ADD_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -293,9 +321,11 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -307,17 +337,21 @@ } case _GUARD_BOTH_UNICODE: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_ADD_UNICODE: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -330,9 +364,11 @@ case BINARY_SUBSCR: { static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -353,10 +389,13 @@ } case BINARY_SLICE: { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + PyObject *stop; + PyObject *start; + PyObject *container; PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -375,10 +414,14 @@ } case STORE_SLICE: { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -396,9 +439,11 @@ } case BINARY_SUBSCR_LIST_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + PyObject *sub; + PyObject *list; PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -418,9 +463,11 @@ } case BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; + PyObject *sub; + PyObject *tuple; PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -440,9 +487,11 @@ } case BINARY_SUBSCR_DICT: { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + PyObject *sub; + PyObject *dict; PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -463,16 +512,20 @@ } case LIST_APPEND: { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); break; } case SET_ADD: { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -482,9 +535,12 @@ case STORE_SUBSCR: { static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -506,9 +562,12 @@ } case STORE_SUBSCR_LIST_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -530,9 +589,12 @@ } case STORE_SUBSCR_DICT: { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -543,8 +605,10 @@ } case DELETE_SUBSCR: { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -555,8 +619,9 @@ } case CALL_INTRINSIC_1: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); @@ -566,9 +631,11 @@ } case CALL_INTRINSIC_2: { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); @@ -580,8 +647,9 @@ } case GET_AITER: { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *iter; + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -617,8 +685,9 @@ } case GET_ANEXT: { - PyObject *aiter = stack_pointer[-1]; + PyObject *aiter; PyObject *awaitable; + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -667,8 +736,9 @@ } case GET_AWAITABLE: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { @@ -697,7 +767,8 @@ } case POP_EXCEPT: { - PyObject *exc_value = stack_pointer[-1]; + PyObject *exc_value; + exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -726,7 +797,8 @@ } case STORE_NAME: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -768,7 +840,8 @@ case UNPACK_SEQUENCE: { static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -789,8 +862,10 @@ } case UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -804,8 +879,10 @@ } case UNPACK_SEQUENCE_TUPLE: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -820,8 +897,10 @@ } case UNPACK_SEQUENCE_LIST: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -836,7 +915,8 @@ } case UNPACK_EX: { - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -848,8 +928,10 @@ case STORE_ATTR: { static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -871,7 +953,8 @@ } case DELETE_ATTR: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); @@ -881,7 +964,8 @@ } case STORE_GLOBAL: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -920,8 +1004,9 @@ } case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict = stack_pointer[-1]; + PyObject *mod_or_class_dict; PyObject *v; + mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { Py_DECREF(mod_or_class_dict); @@ -1004,8 +1089,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1040,8 +1125,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1058,8 +1143,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1085,8 +1170,9 @@ } case LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict = stack_pointer[-1]; + PyObject *class_dict; PyObject *value; + class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -1124,7 +1210,8 @@ } case STORE_DEREF: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1148,8 +1235,9 @@ } case BUILD_STRING: { - PyObject **pieces = (stack_pointer - oparg); + PyObject **pieces; PyObject *str; + pieces = stack_pointer - oparg; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); @@ -1162,8 +1250,9 @@ } case BUILD_TUPLE: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *tup; + values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1173,8 +1262,9 @@ } case BUILD_LIST: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *list; + values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1184,8 +1274,10 @@ } case LIST_EXTEND: { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1206,8 +1298,10 @@ } case SET_UPDATE: { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1216,8 +1310,9 @@ } case BUILD_SET: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *set; + values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) goto error; @@ -1239,8 +1334,9 @@ } case BUILD_MAP: { - PyObject **values = (stack_pointer - oparg*2); + PyObject **values; PyObject *map; + values = stack_pointer - oparg*2; map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1300,9 +1396,11 @@ } case BUILD_CONST_KEY_MAP: { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *keys; + PyObject **values; PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1323,7 +1421,8 @@ } case DICT_UPDATE: { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1340,7 +1439,8 @@ } case DICT_MERGE: { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -1354,8 +1454,10 @@ } case MAP_ADD: { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; + PyObject *value; + PyObject *key; + value = stack_pointer[-1]; + key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -1366,11 +1468,14 @@ } case LOAD_SUPER_ATTR_ATTR: { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1383,17 +1488,20 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case LOAD_SUPER_ATTR_METHOD: { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1417,16 +1525,17 @@ res2 = NULL; } STACK_SHRINK(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; break; } case LOAD_ATTR: { static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1473,13 +1582,14 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case _GUARD_TYPE_VERSION: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1488,7 +1598,8 @@ } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -1497,9 +1608,10 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; @@ -1509,16 +1621,18 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case COMPARE_OP: { static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1546,9 +1660,11 @@ } case COMPARE_OP_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1566,9 +1682,11 @@ } case COMPARE_OP_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -1590,9 +1708,11 @@ } case COMPARE_OP_STR: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1611,9 +1731,11 @@ } case IS_OP: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -1624,9 +1746,11 @@ } case CONTAINS_OP: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); @@ -1638,10 +1762,12 @@ } case CHECK_EG_MATCH: { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; + PyObject *match_type; + PyObject *exc_value; PyObject *rest; PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); @@ -1662,15 +1788,17 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - stack_pointer[-1] = match; stack_pointer[-2] = rest; + stack_pointer[-1] = match; break; } case CHECK_EXC_MATCH: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); @@ -1685,8 +1813,9 @@ } case IS_NONE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *b; + value = stack_pointer[-1]; if (Py_IsNone(value)) { b = Py_True; } @@ -1699,8 +1828,9 @@ } case GET_LEN: { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *len_o; + obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -1712,10 +1842,13 @@ } case MATCH_CLASS: { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -1736,8 +1869,9 @@ } case MATCH_MAPPING: { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -1746,8 +1880,9 @@ } case MATCH_SEQUENCE: { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -1756,9 +1891,11 @@ } case MATCH_KEYS: { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; + PyObject *keys; + PyObject *subject; PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -1768,8 +1905,9 @@ } case GET_ITER: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -1779,8 +1917,9 @@ } case GET_YIELD_FROM_ITER: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -1810,14 +1949,16 @@ } case _ITER_CHECK_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -1838,8 +1979,9 @@ } case _ITER_NEXT_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -1852,14 +1994,16 @@ } case _ITER_CHECK_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -1880,8 +2024,9 @@ } case _ITER_NEXT_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -1894,15 +2039,17 @@ } case _ITER_CHECK_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); exhausted = r->len <= 0 ? Py_True : Py_False; @@ -1912,8 +2059,9 @@ } case _ITER_NEXT_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -1928,10 +2076,13 @@ } case WITH_EXCEPT_START: { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -1963,8 +2114,9 @@ } case PUSH_EXC_INFO: { - PyObject *new_exc = stack_pointer[-1]; + PyObject *new_exc; PyObject *prev_exc; + new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -1975,16 +2127,19 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; break; } case CALL_NO_KW_TYPE_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2001,10 +2156,13 @@ } case CALL_NO_KW_STR_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2023,10 +2181,13 @@ } case CALL_NO_KW_TUPLE_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2045,7 +2206,8 @@ } case EXIT_INIT_CHECK: { - PyObject *should_be_none = stack_pointer[-1]; + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -2058,10 +2220,13 @@ } case CALL_NO_KW_BUILTIN_O: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -2097,10 +2262,13 @@ } case CALL_NO_KW_BUILTIN_FAST: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -2140,10 +2308,13 @@ } case CALL_NO_KW_LEN: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; @@ -2175,10 +2346,13 @@ } case CALL_NO_KW_ISINSTANCE: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -2212,9 +2386,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_O: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -2253,9 +2429,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -2292,9 +2470,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -2330,8 +2510,9 @@ } case MAKE_FUNCTION: { - PyObject *codeobj = stack_pointer[-1]; + PyObject *codeobj; PyObject *func; + codeobj = stack_pointer[-1]; PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -2348,8 +2529,10 @@ } case SET_FUNCTION_ATTRIBUTE: { - PyObject *func = stack_pointer[-1]; - PyObject *attr = stack_pointer[-2]; + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -2380,10 +2563,13 @@ } case BUILD_SLICE: { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -2396,8 +2582,9 @@ } case CONVERT_VALUE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *result; + value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -2409,8 +2596,9 @@ } case FORMAT_SIMPLE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -2426,9 +2614,11 @@ } case FORMAT_WITH_SPEC: { - PyObject *fmt_spec = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *fmt_spec; + PyObject *value; PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -2439,8 +2629,9 @@ } case COPY: { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *bottom; PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); @@ -2450,9 +2641,11 @@ case BINARY_OP: { static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + PyObject *rhs; + PyObject *lhs; PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2476,16 +2669,19 @@ } case SWAP: { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; break; } case _POP_JUMP_IF_FALSE: { - PyObject *flag = stack_pointer[-1]; + PyObject *flag; + flag = stack_pointer[-1]; if (Py_IsFalse(flag)) { pc = oparg; } @@ -2494,7 +2690,8 @@ } case _POP_JUMP_IF_TRUE: { - PyObject *flag = stack_pointer[-1]; + PyObject *flag; + flag = stack_pointer[-1]; if (Py_IsTrue(flag)) { pc = oparg; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9fa549a1203245..7250240ac2396c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -91,8 +91,8 @@ Py_INCREF(value1); Py_INCREF(value2); STACK_GROW(2); - stack_pointer[-1] = value2; stack_pointer[-2] = value1; + stack_pointer[-1] = value2; DISPATCH(); } @@ -106,15 +106,17 @@ } TARGET(STORE_FAST) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_FAST_LOAD_FAST) { - PyObject *value1 = stack_pointer[-1]; + PyObject *value1; PyObject *value2; + value1 = stack_pointer[-1]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); @@ -125,8 +127,10 @@ } TARGET(STORE_FAST_STORE_FAST) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); @@ -136,7 +140,8 @@ } TARGET(POP_TOP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -151,14 +156,15 @@ } TARGET(END_FOR) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *value; + // POP_TOP + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; Py_DECREF(value); } + // POP_TOP + value = stack_pointer[-2]; { - PyObject *value = _tmp_2; Py_DECREF(value); } STACK_SHRINK(2); @@ -166,8 +172,10 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -184,8 +192,10 @@ } TARGET(END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; Py_DECREF(receiver); STACK_SHRINK(1); stack_pointer[-1] = value; @@ -193,8 +203,10 @@ } TARGET(INSTRUMENTED_END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -209,8 +221,9 @@ } TARGET(UNARY_NEGATIVE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -219,8 +232,9 @@ } TARGET(UNARY_NOT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; @@ -230,8 +244,9 @@ TARGET(TO_BOOL) { PREDICTED(TO_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -252,7 +267,8 @@ } TARGET(TO_BOOL_BOOL) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); next_instr += 3; @@ -260,8 +276,9 @@ } TARGET(TO_BOOL_INT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { @@ -278,8 +295,9 @@ } TARGET(TO_BOOL_LIST) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; @@ -290,8 +308,9 @@ } TARGET(TO_BOOL_NONE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -302,8 +321,9 @@ } TARGET(TO_BOOL_STR) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { @@ -321,8 +341,9 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; uint32_t version = read_u32(&next_instr[1].cache); // This one is a bit weird, because we expect *some* failures: assert(version); @@ -336,8 +357,9 @@ } TARGET(UNARY_INVERT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -346,215 +368,192 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_MULTIPLY_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_SUBTRACT_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_MULTIPLY_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_SUBTRACT_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_UNICODE { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_INPLACE_ADD_UNICODE { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); @@ -586,9 +585,11 @@ TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -610,10 +611,13 @@ } TARGET(BINARY_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + PyObject *stop; + PyObject *start; + PyObject *container; PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -632,10 +636,14 @@ } TARGET(STORE_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -653,9 +661,11 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + PyObject *sub; + PyObject *list; PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -676,9 +686,11 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; + PyObject *sub; + PyObject *tuple; PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -699,9 +711,11 @@ } TARGET(BINARY_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + PyObject *sub; + PyObject *dict; PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -723,8 +737,10 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); @@ -747,19 +763,24 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(1); } TARGET(LIST_APPEND) { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); DISPATCH(); } TARGET(SET_ADD) { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -770,9 +791,12 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -795,9 +819,12 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -820,9 +847,12 @@ } TARGET(STORE_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -834,8 +864,10 @@ } TARGET(DELETE_SUBSCR) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -846,8 +878,9 @@ } TARGET(CALL_INTRINSIC_1) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); @@ -857,9 +890,11 @@ } TARGET(CALL_INTRINSIC_2) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); @@ -871,7 +906,8 @@ } TARGET(RAISE_VARARGS) { - PyObject **args = (stack_pointer - oparg); + PyObject **args; + args = stack_pointer - oparg; PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -893,10 +929,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } + STACK_SHRINK(oparg); } TARGET(INTERPRETER_EXIT) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous cframe and return. */ @@ -905,10 +943,12 @@ assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; + STACK_SHRINK(1); } TARGET(RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -921,10 +961,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + STACK_SHRINK(1); } TARGET(INSTRUMENTED_RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -941,6 +983,7 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + STACK_SHRINK(1); } TARGET(RETURN_CONST) { @@ -980,8 +1023,9 @@ } TARGET(GET_AITER) { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *iter; + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1017,8 +1061,9 @@ } TARGET(GET_ANEXT) { - PyObject *aiter = stack_pointer[-1]; + PyObject *aiter; PyObject *awaitable; + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1067,8 +1112,9 @@ } TARGET(GET_AWAITABLE) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { @@ -1099,9 +1145,11 @@ TARGET(SEND) { PREDICTED(SEND); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *v; + PyObject *receiver; PyObject *retval; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1154,8 +1202,10 @@ } TARGET(SEND_GEN) { - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && @@ -1174,7 +1224,8 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ PyGenObject *gen = _PyFrame_GetGenerator(frame); @@ -1195,7 +1246,8 @@ } TARGET(YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1215,7 +1267,8 @@ } TARGET(POP_EXCEPT) { - PyObject *exc_value = stack_pointer[-1]; + PyObject *exc_value; + exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -1223,8 +1276,10 @@ } TARGET(RERAISE) { - PyObject *exc = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1243,11 +1298,14 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, next_instr-1); goto exception_unwind; + STACK_SHRINK(1); } TARGET(END_ASYNC_FOR) { - PyObject *exc = stack_pointer[-1]; - PyObject *awaitable = stack_pointer[-2]; + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -1264,11 +1322,14 @@ } TARGET(CLEANUP_THROW) { - PyObject *exc_value = stack_pointer[-1]; - PyObject *last_sent_val = stack_pointer[-2]; - PyObject *sub_iter = stack_pointer[-3]; + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; PyObject *none; PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1284,8 +1345,8 @@ goto exception_unwind; } STACK_SHRINK(1); - stack_pointer[-1] = value; stack_pointer[-2] = none; + stack_pointer[-1] = value; DISPATCH(); } @@ -1311,7 +1372,8 @@ } TARGET(STORE_NAME) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -1354,7 +1416,8 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1376,8 +1439,10 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -1392,8 +1457,10 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1409,8 +1476,10 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1426,7 +1495,8 @@ } TARGET(UNPACK_EX) { - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -1439,8 +1509,10 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1463,7 +1535,8 @@ } TARGET(DELETE_ATTR) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); @@ -1473,7 +1546,8 @@ } TARGET(STORE_GLOBAL) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -1498,27 +1572,25 @@ } TARGET(LOAD_LOCALS) { - PyObject *_tmp_1; - { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - _tmp_1 = locals; + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } + Py_INCREF(locals); STACK_GROW(1); - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = locals; DISPATCH(); } TARGET(LOAD_NAME) { - PyObject *_tmp_1; + PyObject *locals; + PyObject *mod_or_class_dict; + PyObject *v; + // _LOAD_LOCALS { - PyObject *locals; locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1526,11 +1598,10 @@ if (true) goto error; } Py_INCREF(locals); - _tmp_1 = locals; } + // _LOAD_FROM_DICT_OR_GLOBALS + mod_or_class_dict = locals; { - PyObject *mod_or_class_dict = _tmp_1; - PyObject *v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { Py_DECREF(mod_or_class_dict); @@ -1557,47 +1628,43 @@ } } } - _tmp_1 = v; } STACK_GROW(1); - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = v; DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *_tmp_1 = stack_pointer[-1]; - { - PyObject *mod_or_class_dict = _tmp_1; - PyObject *v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + Py_DECREF(mod_or_class_dict); + goto error; + } + Py_DECREF(mod_or_class_dict); + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { goto error; } - Py_DECREF(mod_or_class_dict); - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { goto error; } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; - } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; } } - _tmp_1 = v; } - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = v; DISPATCH(); } @@ -1654,15 +1721,16 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - PyObject *_tmp_1; - PyObject *_tmp_2; + PyObject *null = NULL; + PyObject *res; + // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1670,9 +1738,8 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _LOAD_GLOBAL_MODULE { - PyObject *null = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -1681,20 +1748,19 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - if (oparg & 1) { _tmp_2 = null; } - _tmp_1 = res; } - next_instr += 4; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *_tmp_1; - PyObject *_tmp_2; + PyObject *null = NULL; + PyObject *res; + // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1702,6 +1768,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); @@ -1709,9 +1776,8 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _LOAD_GLOBAL_BUILTINS { - PyObject *null = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); @@ -1720,14 +1786,12 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - if (oparg & 1) { _tmp_2 = null; } - _tmp_1 = res; } - next_instr += 4; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } @@ -1765,8 +1829,9 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { - PyObject *class_dict = stack_pointer[-1]; + PyObject *class_dict; PyObject *value; + class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -1804,7 +1869,8 @@ } TARGET(STORE_DEREF) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1828,8 +1894,9 @@ } TARGET(BUILD_STRING) { - PyObject **pieces = (stack_pointer - oparg); + PyObject **pieces; PyObject *str; + pieces = stack_pointer - oparg; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); @@ -1842,8 +1909,9 @@ } TARGET(BUILD_TUPLE) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *tup; + values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1853,8 +1921,9 @@ } TARGET(BUILD_LIST) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *list; + values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1864,8 +1933,10 @@ } TARGET(LIST_EXTEND) { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1886,8 +1957,10 @@ } TARGET(SET_UPDATE) { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1896,8 +1969,9 @@ } TARGET(BUILD_SET) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *set; + values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) goto error; @@ -1919,8 +1993,9 @@ } TARGET(BUILD_MAP) { - PyObject **values = (stack_pointer - oparg*2); + PyObject **values; PyObject *map; + values = stack_pointer - oparg*2; map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1980,9 +2055,11 @@ } TARGET(BUILD_CONST_KEY_MAP) { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *keys; + PyObject **values; PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2003,7 +2080,8 @@ } TARGET(DICT_UPDATE) { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2020,7 +2098,8 @@ } TARGET(DICT_MERGE) { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -2034,8 +2113,10 @@ } TARGET(MAP_ADD) { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; + PyObject *value; + PyObject *key; + value = stack_pointer[-1]; + key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -2051,16 +2132,21 @@ // don't want to specialize instrumented instructions INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_SUPER_ATTR) { PREDICTED(LOAD_SUPER_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -2111,18 +2197,21 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2135,18 +2224,21 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2170,8 +2262,8 @@ res2 = NULL; } STACK_SHRINK(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 1; DISPATCH(); } @@ -2179,9 +2271,10 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2228,35 +2321,33 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *owner; + PyObject *res2 = NULL; + PyObject *res; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - PyObject *owner = _tmp_2; uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - _tmp_2 = owner; } + // _CHECK_MANAGED_OBJECT_HAS_VALUES { - PyObject *owner = _tmp_2; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - _tmp_2 = owner; } + // _LOAD_ATTR_INSTANCE_VALUE { - PyObject *owner = _tmp_2; - PyObject *res2 = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; @@ -2265,20 +2356,19 @@ Py_INCREF(res); res2 = NULL; Py_DECREF(owner); - if (oparg & 1) { _tmp_2 = res2; } - _tmp_1 = res; } - next_instr += 9; STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } + stack_pointer[-1] = res; + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); @@ -2295,16 +2385,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2335,16 +2426,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2358,16 +2450,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *cls = stack_pointer[-1]; + PyObject *cls; PyObject *res2 = NULL; PyObject *res; + cls = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -2383,14 +2476,15 @@ Py_INCREF(res); Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); @@ -2417,10 +2511,12 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); @@ -2449,11 +2545,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(STORE_ATTR_INSTANCE_VALUE) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2479,8 +2578,10 @@ } TARGET(STORE_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2527,8 +2628,10 @@ } TARGET(STORE_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2548,9 +2651,11 @@ TARGET(COMPARE_OP) { PREDICTED(COMPARE_OP); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2579,9 +2684,11 @@ } TARGET(COMPARE_OP_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2600,9 +2707,11 @@ } TARGET(COMPARE_OP_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2625,9 +2734,11 @@ } TARGET(COMPARE_OP_STR) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2647,9 +2758,11 @@ } TARGET(IS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -2660,9 +2773,11 @@ } TARGET(CONTAINS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); @@ -2674,10 +2789,12 @@ } TARGET(CHECK_EG_MATCH) { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; + PyObject *match_type; + PyObject *exc_value; PyObject *rest; PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); @@ -2698,15 +2815,17 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - stack_pointer[-1] = match; stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } TARGET(CHECK_EXC_MATCH) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); @@ -2721,9 +2840,11 @@ } TARGET(IMPORT_NAME) { - PyObject *fromlist = stack_pointer[-1]; - PyObject *level = stack_pointer[-2]; + PyObject *fromlist; + PyObject *level; PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2735,8 +2856,9 @@ } TARGET(IMPORT_FROM) { - PyObject *from = stack_pointer[-1]; + PyObject *from; PyObject *res; + from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2797,7 +2919,8 @@ } TARGET(POP_JUMP_IF_FALSE) { - PyObject *cond = stack_pointer[-1]; + PyObject *cond; + cond = stack_pointer[-1]; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); STACK_SHRINK(1); @@ -2805,7 +2928,8 @@ } TARGET(POP_JUMP_IF_TRUE) { - PyObject *cond = stack_pointer[-1]; + PyObject *cond; + cond = stack_pointer[-1]; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); STACK_SHRINK(1); @@ -2813,10 +2937,12 @@ } TARGET(POP_JUMP_IF_NONE) { - PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *value; + PyObject *b; + PyObject *cond; + // IS_NONE + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; - PyObject *b; if (Py_IsNone(value)) { b = Py_True; } @@ -2824,10 +2950,10 @@ b = Py_False; Py_DECREF(value); } - _tmp_1 = b; } + // POP_JUMP_IF_TRUE + cond = b; { - PyObject *cond = _tmp_1; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); } @@ -2836,10 +2962,12 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *value; + PyObject *b; + PyObject *cond; + // IS_NONE + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; - PyObject *b; if (Py_IsNone(value)) { b = Py_True; } @@ -2847,10 +2975,10 @@ b = Py_False; Py_DECREF(value); } - _tmp_1 = b; } + // POP_JUMP_IF_FALSE + cond = b; { - PyObject *cond = _tmp_1; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); } @@ -2869,8 +2997,9 @@ } TARGET(GET_LEN) { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *len_o; + obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -2882,10 +3011,13 @@ } TARGET(MATCH_CLASS) { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -2906,8 +3038,9 @@ } TARGET(MATCH_MAPPING) { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -2916,8 +3049,9 @@ } TARGET(MATCH_SEQUENCE) { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -2926,9 +3060,11 @@ } TARGET(MATCH_KEYS) { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; + PyObject *keys; + PyObject *subject; PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -2938,8 +3074,9 @@ } TARGET(GET_ITER) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -2949,8 +3086,9 @@ } TARGET(GET_YIELD_FROM_ITER) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2982,8 +3120,9 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3051,15 +3190,15 @@ } TARGET(FOR_ITER_LIST) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_LIST { - PyObject *iter = _tmp_2; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); @@ -3076,37 +3215,32 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_LIST { - PyObject *iter = _tmp_2; - PyObject *next; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_TUPLE { - PyObject *iter = _tmp_2; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); @@ -3123,38 +3257,33 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_TUPLE { - PyObject *iter = _tmp_2; - PyObject *next; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_RANGE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_RANGE { - PyObject *iter = _tmp_2; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); @@ -3166,11 +3295,9 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_RANGE { - PyObject *iter = _tmp_2; - PyObject *next; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -3179,18 +3306,16 @@ r->len--; next = PyLong_FromLong(value); if (next == NULL) goto error; - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_GEN) { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3206,12 +3331,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); + STACK_GROW(1); } TARGET(BEFORE_ASYNC_WITH) { - PyObject *mgr = stack_pointer[-1]; + PyObject *mgr; PyObject *exit; PyObject *res; + mgr = stack_pointer[-1]; PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3242,15 +3369,16 @@ if (true) goto pop_1_error; } STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } TARGET(BEFORE_WITH) { - PyObject *mgr = stack_pointer[-1]; + PyObject *mgr; PyObject *exit; PyObject *res; + mgr = stack_pointer[-1]; /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3284,16 +3412,19 @@ if (true) goto pop_1_error; } STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } TARGET(WITH_EXCEPT_START) { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3325,8 +3456,9 @@ } TARGET(PUSH_EXC_INFO) { - PyObject *new_exc = stack_pointer[-1]; + PyObject *new_exc; PyObject *prev_exc; + new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3337,15 +3469,16 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3366,16 +3499,17 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); @@ -3388,15 +3522,16 @@ res2 = Py_NewRef(descr); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; + PyObject *self; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3420,8 +3555,9 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PyObject *self = stack_pointer[-1]; + PyObject *self; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); @@ -3439,9 +3575,10 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); @@ -3458,8 +3595,8 @@ res2 = Py_NewRef(descr); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } @@ -3489,10 +3626,13 @@ TARGET(CALL) { PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3583,8 +3723,10 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *callable; + PyObject *method; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3594,13 +3736,18 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3627,12 +3774,17 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3669,13 +3821,18 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_NO_KW_TYPE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3693,10 +3850,13 @@ } TARGET(CALL_NO_KW_STR_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3716,10 +3876,13 @@ } TARGET(CALL_NO_KW_TUPLE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3739,9 +3902,12 @@ } TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -3792,10 +3958,13 @@ * as it will be checked after start_frame */ tstate->py_recursion_remaining--; goto start_frame; + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(EXIT_INIT_CHECK) { - PyObject *should_be_none = stack_pointer[-1]; + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -3808,10 +3977,13 @@ } TARGET(CALL_BUILTIN_CLASS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3842,10 +4014,13 @@ } TARGET(CALL_NO_KW_BUILTIN_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -3882,10 +4057,13 @@ } TARGET(CALL_NO_KW_BUILTIN_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -3926,10 +4104,13 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3970,10 +4151,13 @@ } TARGET(CALL_NO_KW_LEN) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; @@ -4006,10 +4190,13 @@ } TARGET(CALL_NO_KW_ISINSTANCE) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4044,9 +4231,12 @@ } TARGET(CALL_NO_KW_LIST_APPEND) { - PyObject **args = (stack_pointer - oparg); - PyObject *self = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *self; + PyObject *method; + args = stack_pointer - oparg; + self = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); assert(method != NULL); @@ -4064,12 +4254,16 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -4109,9 +4303,11 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4149,9 +4345,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4189,9 +4387,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -4233,10 +4433,13 @@ TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); - PyObject *kwargs = (oparg & 1) ? stack_pointer[-(((oparg & 1) ? 1 : 0))] : NULL; - PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; - PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } + callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; + func = stack_pointer[-2 - (oparg & 1 ? 1 : 0)]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4311,8 +4514,9 @@ } TARGET(MAKE_FUNCTION) { - PyObject *codeobj = stack_pointer[-1]; + PyObject *codeobj; PyObject *func; + codeobj = stack_pointer[-1]; PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4329,8 +4533,10 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { - PyObject *func = stack_pointer[-1]; - PyObject *attr = stack_pointer[-2]; + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4384,10 +4590,13 @@ } TARGET(BUILD_SLICE) { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -4400,8 +4609,9 @@ } TARGET(CONVERT_VALUE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *result; + value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -4413,8 +4623,9 @@ } TARGET(FORMAT_SIMPLE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4430,9 +4641,11 @@ } TARGET(FORMAT_WITH_SPEC) { - PyObject *fmt_spec = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *fmt_spec; + PyObject *value; PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -4443,8 +4656,9 @@ } TARGET(COPY) { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *bottom; PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); @@ -4455,9 +4669,11 @@ TARGET(BINARY_OP) { PREDICTED(BINARY_OP); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + PyObject *rhs; + PyObject *lhs; PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4482,11 +4698,13 @@ } TARGET(SWAP) { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 53ae7360b8ae12..bd8918a87ffe08 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -13,7 +13,6 @@ MacroParts, OverriddenInstructionPlaceHolder, PseudoInstruction, - StackEffectMapping, ) import parsing from parsing import StackEffect @@ -34,11 +33,12 @@ class Analyzer: input_filenames: list[str] errors: int = 0 + warnings: int = 0 def __init__(self, input_filenames: list[str]): self.input_filenames = input_filenames - def error(self, msg: str, node: parsing.Node) -> None: + def message(self, msg: str, node: parsing.Node) -> None: lineno = 0 filename = "" if context := node.context: @@ -49,8 +49,18 @@ def error(self, msg: str, node: parsing.Node) -> None: if token.kind != "COMMENT": break print(f"{filename}:{lineno}: {msg}", file=sys.stderr) + + def error(self, msg: str, node: parsing.Node) -> None: + self.message("error: " + msg, node) self.errors += 1 + def warning(self, msg: str, node: parsing.Node) -> None: + self.message("warning: " + msg, node) + self.warnings += 1 + + def note(self, msg: str, node: parsing.Node) -> None: + self.message("note: " + msg, node) + everything: list[ parsing.InstDef | parsing.Macro @@ -83,8 +93,15 @@ def parse(self) -> None: self.parse_file(filename, instrs_idx) files = " + ".join(self.input_filenames) + n_instrs = 0 + n_ops = 0 + for instr in self.instrs.values(): + if instr.kind == "op": + n_ops += 1 + else: + n_instrs += 1 print( - f"Read {len(self.instrs)} instructions/ops, " + f"Read {n_instrs} instructions, {n_ops} ops, " f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " f"and {len(self.families)} families from {files}", file=sys.stderr, @@ -270,14 +287,70 @@ def analyze_macros_and_pseudos(self) -> None: self.macro_instrs = {} self.pseudo_instrs = {} for name, macro in self.macros.items(): - self.macro_instrs[name] = self.analyze_macro(macro) + self.macro_instrs[name] = mac = self.analyze_macro(macro) + self.check_macro_consistency(mac) for name, pseudo in self.pseudos.items(): self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) + # TODO: Merge with similar code in stacking.py, write_components() + def check_macro_consistency(self, mac: MacroInstruction) -> None: + def get_var_names(instr: Instruction) -> dict[str, StackEffect]: + vars: dict[str, StackEffect] = {} + for eff in instr.input_effects + instr.output_effects: + if eff.name in vars: + if vars[eff.name] != eff: + self.error( + f"Instruction {instr.name!r} has " + f"inconsistent type/cond/size for variable " + f"{eff.name!r}: {vars[eff.name]} vs {eff}", + instr.inst, + ) + else: + vars[eff.name] = eff + return vars + + all_vars: dict[str, StackEffect] = {} + # print("Checking", mac.name) + prevop: Instruction | None = None + for part in mac.parts: + if not isinstance(part, Component): + continue + vars = get_var_names(part.instr) + # print(" //", part.instr.name, "//", vars) + for name, eff in vars.items(): + if name in all_vars: + if all_vars[name] != eff: + self.error( + f"Macro {mac.name!r} has " + f"inconsistent type/cond/size for variable " + f"{name!r}: " + f"{all_vars[name]} vs {eff} in {part.instr.name!r}", + mac.macro, + ) + else: + all_vars[name] = eff + if prevop is not None: + pushes = list(prevop.output_effects) + pops = list(reversed(part.instr.input_effects)) + copies: list[tuple[StackEffect, StackEffect]] = [] + while pushes and pops and pushes[-1] == pops[0]: + src, dst = pushes.pop(), pops.pop(0) + if src.name == dst.name or dst.name is UNUSED: + continue + copies.append((src, dst)) + reads = set(copy[0].name for copy in copies) + writes = set(copy[1].name for copy in copies) + if reads & writes: + self.error( + f"Macro {mac.name!r} has conflicting copies " + f"(source of one copy is destination of another): " + f"{reads & writes}", + mac.macro, + ) + prevop = part.instr + def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: components = self.check_macro_components(macro) - stack, initial_sp = self.stack_analysis(components) - sp = initial_sp parts: MacroParts = [] flags = InstructionFlags.newEmpty() offset = 0 @@ -287,20 +360,15 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: parts.append(ceffect) offset += ceffect.size case Instruction() as instr: - part, sp, offset = self.analyze_instruction( - instr, stack, sp, offset - ) + part, offset = self.analyze_instruction(instr, offset) parts.append(part) flags.add(instr.instr_flags) case _: typing.assert_never(component) - final_sp = sp format = "IB" if offset: format += "C" + "0" * (offset - 1) - return MacroInstruction( - macro.name, stack, initial_sp, final_sp, format, flags, macro, parts, offset - ) + return MacroInstruction(macro.name, format, flags, macro, parts, offset) def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: targets = [self.instrs[target] for target in pseudo.targets] @@ -312,24 +380,15 @@ def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) def analyze_instruction( - self, instr: Instruction, stack: list[StackEffect], sp: int, offset: int - ) -> tuple[Component, int, int]: - input_mapping: StackEffectMapping = [] - for ieffect in reversed(instr.input_effects): - sp -= 1 - input_mapping.append((stack[sp], ieffect)) - output_mapping: StackEffectMapping = [] - for oeffect in instr.output_effects: - output_mapping.append((stack[sp], oeffect)) - sp += 1 + self, instr: Instruction, offset: int + ) -> tuple[Component, int]: active_effects: list[ActiveCacheEffect] = [] for ceffect in instr.cache_effects: if ceffect.name != UNUSED: active_effects.append(ActiveCacheEffect(ceffect, offset)) offset += ceffect.size return ( - Component(instr, input_mapping, output_mapping, active_effects), - sp, + Component(instr, active_effects), offset, ) @@ -348,65 +407,3 @@ def check_macro_components( case _: typing.assert_never(uop) return components - - def stack_analysis( - self, components: typing.Iterable[InstructionOrCacheEffect] - ) -> tuple[list[StackEffect], int]: - """Analyze a macro. - - Ignore cache effects. - - Return the list of variables (as StackEffects) and the initial stack pointer. - """ - lowest = current = highest = 0 - conditions: dict[int, str] = {} # Indexed by 'current'. - last_instr: Instruction | None = None - for thing in components: - if isinstance(thing, Instruction): - last_instr = thing - for thing in components: - match thing: - case Instruction() as instr: - if any( - eff.size for eff in instr.input_effects + instr.output_effects - ): - # TODO: Eventually this will be needed, at least for macros. - self.error( - f"Instruction {instr.name!r} has variable-sized stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if any(eff.cond for eff in instr.input_effects): - self.error( - f"Instruction {instr.name!r} has conditional input stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if ( - any(eff.cond for eff in instr.output_effects) - and instr is not last_instr - ): - self.error( - f"Instruction {instr.name!r} has conditional output stack effect, " - "but is not the last instruction in a macro", - instr.inst, # TODO: Pass name+location of macro - ) - current -= len(instr.input_effects) - lowest = min(lowest, current) - for eff in instr.output_effects: - if eff.cond: - conditions[current] = eff.cond - current += 1 - highest = max(highest, current) - case parsing.CacheEffect(): - pass - case _: - typing.assert_never(thing) - # At this point, 'current' is the net stack effect, - # and 'lowest' and 'highest' are the extremes. - # Note that 'lowest' may be negative. - stack = [ - StackEffect(f"_tmp_{i}", "", conditions.get(highest - i, "")) - for i in reversed(range(1, highest - lowest + 1)) - ] - return stack, -lowest diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 78acd93b73ac31..f7ebdeb0d65677 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -49,9 +49,9 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value=None): + def names(self, value=None) -> list[str]: if value is None: - return dataclasses.asdict(self).keys() + return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] def bitmap(self) -> int: diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index 6b5a375af891be..5894751bd9635c 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -2,7 +2,7 @@ import re import typing -from parsing import StackEffect +from parsing import StackEffect, Family UNUSED = "unused" @@ -19,8 +19,11 @@ class Formatter: nominal_filename: str def __init__( - self, stream: typing.TextIO, indent: int, - emit_line_directives: bool = False, comment: str = "//", + self, + stream: typing.TextIO, + indent: int, + emit_line_directives: bool = False, + comment: str = "//", ) -> None: self.stream = stream self.prefix = " " * indent @@ -93,8 +96,11 @@ def declare(self, dst: StackEffect, src: StackEffect | None): typ = f"{dst.type}" if dst.type else "PyObject *" if src: cast = self.cast(dst, src) - init = f" = {cast}{src.name}" - elif dst.cond: + initexpr = f"{cast}{src.name}" + if src.cond and src.cond != "1": + initexpr = f"{parenthesize_cond(src.cond)} ? {initexpr} : NULL" + init = f" = {initexpr}" + elif dst.cond and dst.cond != "1": init = " = NULL" else: init = "" @@ -102,10 +108,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None): self.emit(f"{typ}{sepa}{dst.name}{init};") def assign(self, dst: StackEffect, src: StackEffect): - if src.name == UNUSED: - return - if src.size: - # Don't write sized arrays -- it's up to the user code. + if src.name == UNUSED or dst.name == UNUSED: return cast = self.cast(dst, src) if re.match(r"^REG\(oparg(\d+)\)$", dst.name): @@ -122,6 +125,23 @@ def assign(self, dst: StackEffect, src: StackEffect): def cast(self, dst: StackEffect, src: StackEffect) -> str: return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" + def static_assert_family_size( + self, name: str, family: Family | None, cache_offset: int + ) -> None: + """Emit a static_assert for the size of a family, if known. + + This will fail at compile time if the cache size computed from + the instruction definition does not match the size of the struct + used by specialize.c. + """ + if family and name == family.name: + cache_size = family.size + if cache_size: + self.emit( + f"static_assert({cache_size} == {cache_offset}, " + f'"incorrect cache size");' + ) + def prettify_filename(filename: str) -> str: # Make filename more user-friendly and less platform-specific, @@ -178,11 +198,8 @@ def maybe_parenthesize(sym: str) -> str: return f"({sym})" -def string_effect_size(arg: tuple[int, str]) -> str: - numeric, symbolic = arg - if numeric and symbolic: - return f"{numeric} + {symbolic}" - elif symbolic: - return symbolic - else: - return str(numeric) +def parenthesize_cond(cond: str) -> str: + """Parenthesize a condition, but only if it contains ?: itself.""" + if "?" in cond: + cond = f"({cond})" + return cond diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 967e1e2f5b63bb..d35a16a80e8d00 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -4,14 +4,14 @@ """ import argparse -import contextlib import os import posixpath import sys import typing +import stacking # Early import to avoid circular import from analysis import Analyzer -from formatting import Formatter, list_effect_size, maybe_parenthesize +from formatting import Formatter, list_effect_size from flags import InstructionFlags, variable_used from instructions import ( AnyInstruction, @@ -118,41 +118,7 @@ def effect_str(effects: list[StackEffect]) -> str: pushed = "" case parsing.Macro(): instr = self.macro_instrs[thing.name] - parts = [comp for comp in instr.parts if isinstance(comp, Component)] - # Note: stack_analysis() already verifies that macro components - # have no variable-sized stack effects. - low = 0 - sp = 0 - high = 0 - pushed_symbolic: list[str] = [] - for comp in parts: - for effect in comp.instr.input_effects: - assert not effect.cond, effect - assert not effect.size, effect - sp -= 1 - low = min(low, sp) - for effect in comp.instr.output_effects: - assert not effect.size, effect - if effect.cond: - if effect.cond in ("0", "1"): - pushed_symbolic.append(effect.cond) - else: - pushed_symbolic.append( - maybe_parenthesize( - f"{maybe_parenthesize(effect.cond)} ? 1 : 0" - ) - ) - sp += 1 - high = max(sp, high) - if high != max(0, sp): - # If you get this, intermediate stack growth occurs, - # and stack size calculations may go awry. - # E.g. [push, pop]. The fix would be for stack size - # calculations to use the micro ops. - self.error("Macro has virtual stack growth", thing) - popped = str(-low) - pushed_symbolic.append(str(sp - low - len(pushed_symbolic))) - pushed = " + ".join(pushed_symbolic) + popped, pushed = stacking.get_stack_effect_info_for_macro(instr) case parsing.Pseudo(): instr = self.pseudo_instrs[thing.name] popped = pushed = None @@ -258,7 +224,8 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case _: typing.assert_never(thing) all_formats.add(format) - # Turn it into a list of enum definitions. + + # Turn it into a sorted list of enum values. format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] with open(metadata_filename, "w") as f: @@ -276,8 +243,10 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_stack_effect_functions() - # Write type definitions - self.out.emit(f"enum InstructionFormat {{ {', '.join(format_enums)} }};") + # Write the enum definition for instruction formats. + with self.out.block("enum InstructionFormat", ";"): + for enum in format_enums: + self.out.emit(enum + ",") self.out.emit("") self.out.emit( @@ -374,7 +343,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Since an 'op' is not a bytecode, it has no expansion; but 'inst' is if instr.kind == "inst" and instr.is_viable_uop(): # Construct a dummy Component -- input/output mappings are not used - part = Component(instr, [], [], instr.active_caches) + part = Component(instr, instr.active_caches) self.write_macro_expansions(instr.name, [part]) elif instr.kind == "inst" and variable_used( instr.inst, "oparg1" @@ -468,7 +437,15 @@ def write_macro_expansions(self, name: str, parts: MacroParts) -> None: if isinstance(part, Component): # All component instructions must be viable uops if not part.instr.is_viable_uop(): - print(f"NOTE: Part {part.instr.name} of {name} is not a viable uop") + # This note just reminds us about macros that cannot + # be expanded to Tier 2 uops. It is not an error. + # It is sometimes emitted for macros that have a + # manual translation in translate_bytecode_to_trace() + # in Python/optimizer.c. + self.note( + f"Part {part.instr.name} of {name} is not a viable uop", + part.instr.inst, + ) return if not part.active_caches: size, offset = OPARG_SIZES["OPARG_FULL"], 0 @@ -512,7 +489,7 @@ def write_super_expansions(self, name: str) -> None: instr2 = self.instrs[name2] assert not instr1.active_caches, f"{name1} has active caches" assert not instr2.active_caches, f"{name2} has active caches" - expansions = [ + expansions: list[tuple[str, int, int]] = [ (name1, OPARG_SIZES["OPARG_TOP"], 0), (name2, OPARG_SIZES["OPARG_BOTTOM"], 0), ] @@ -563,7 +540,6 @@ def write_instructions( # Write and count instructions of all kinds n_instrs = 0 n_macros = 0 - n_pseudos = 0 for thing in self.everything: match thing: case OverriddenInstructionPlaceHolder(): @@ -574,15 +550,17 @@ def write_instructions( self.write_instr(self.instrs[thing.name]) case parsing.Macro(): n_macros += 1 - self.write_macro(self.macro_instrs[thing.name]) + mac = self.macro_instrs[thing.name] + stacking.write_macro_instr(mac, self.out, self.families.get(mac.name)) + # self.write_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): - n_pseudos += 1 + pass case _: typing.assert_never(thing) print( - f"Wrote {n_instrs} instructions, {n_macros} macros, " - f"and {n_pseudos} pseudos to {output_filename}", + f"Wrote {n_instrs} instructions and {n_macros} macros " + f"to {output_filename}", file=sys.stderr, ) @@ -590,6 +568,8 @@ def write_executor_instructions( self, executor_filename: str, emit_line_directives: bool ) -> None: """Generate cases for the Tier 2 interpreter.""" + n_instrs = 0 + n_uops = 0 with open(executor_filename, "w") as f: self.out = Formatter(f, 8, emit_line_directives) self.write_provenance_header() @@ -601,6 +581,10 @@ def write_executor_instructions( case parsing.InstDef(): instr = self.instrs[thing.name] if instr.is_viable_uop(): + if instr.kind == "op": + n_uops += 1 + else: + n_instrs += 1 self.out.emit("") with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) @@ -616,7 +600,7 @@ def write_executor_instructions( case _: typing.assert_never(thing) print( - f"Wrote some stuff to {executor_filename}", + f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}", file=sys.stderr, ) @@ -642,69 +626,6 @@ def write_instr(self, instr: Instruction) -> None: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit(f"DISPATCH();") - def write_macro(self, mac: MacroInstruction) -> None: - """Write code for a macro instruction.""" - last_instr: Instruction | None = None - with self.wrap_macro(mac): - cache_adjust = 0 - for part in mac.parts: - match part: - case parsing.CacheEffect(size=size): - cache_adjust += size - case Component() as comp: - last_instr = comp.instr - comp.write_body(self.out) - cache_adjust += comp.instr.cache_offset - - if cache_adjust: - self.out.emit(f"next_instr += {cache_adjust};") - - if ( - (family := self.families.get(mac.name)) - and mac.name == family.name - and (cache_size := family.size) - ): - self.out.emit( - f"static_assert({cache_size} == " - f'{cache_adjust}, "incorrect cache size");' - ) - - @contextlib.contextmanager - def wrap_macro(self, mac: MacroInstruction): - """Boilerplate for macro instructions.""" - # TODO: Somewhere (where?) make it so that if one instruction - # has an output that is input to another, and the variable names - # and types match and don't conflict with other instructions, - # that variable is declared with the right name and type in the - # outer block, rather than trusting the compiler to optimize it. - self.out.emit("") - with self.out.block(f"TARGET({mac.name})"): - if mac.predicted: - self.out.emit(f"PREDICTED({mac.name});") - - # The input effects should have no conditionals. - # Only the output effects do (for now). - ieffects = [ - StackEffect(eff.name, eff.type) if eff.cond else eff - for eff in mac.stack - ] - - for i, var in reversed(list(enumerate(ieffects))): - src = None - if i < mac.initial_sp: - src = StackEffect(f"stack_pointer[-{mac.initial_sp - i}]", "") - self.out.declare(var, src) - - yield - - self.out.stack_adjust(ieffects[: mac.initial_sp], mac.stack[: mac.final_sp]) - - for i, var in enumerate(reversed(mac.stack[: mac.final_sp]), 1): - dst = StackEffect(f"stack_pointer[-{i}]", "") - self.out.assign(dst, var) - - self.out.emit(f"DISPATCH();") - def main(): """Parse command line, parse input, analyze, write output.""" diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 6f42699d900b46..aa94dbb07ea1c0 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -2,17 +2,16 @@ import re import typing -from flags import InstructionFlags, variable_used_unspecialized +from flags import InstructionFlags, variable_used, variable_used_unspecialized from formatting import ( Formatter, UNUSED, - string_effect_size, list_effect_size, - maybe_parenthesize, ) import lexer as lx import parsing from parsing import StackEffect +import stacking BITS_PER_CODE_UNIT = 16 @@ -61,6 +60,7 @@ class Instruction: # Computed by constructor always_exits: bool + has_deopt: bool cache_offset: int cache_effects: list[parsing.CacheEffect] input_effects: list[StackEffect] @@ -83,6 +83,7 @@ def __init__(self, inst: parsing.InstDef): self.block ) self.always_exits = always_exits(self.block_text) + self.has_deopt = variable_used(self.inst, "DEOPT_IF") self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] @@ -93,7 +94,7 @@ def __init__(self, inst: parsing.InstDef): self.output_effects = inst.outputs # For consistency/completeness unmoved_names: set[str] = set() for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect.name == oeffect.name: + if ieffect == oeffect and ieffect.name == oeffect.name: unmoved_names.add(ieffect.name) else: break @@ -141,84 +142,17 @@ def is_viable_uop(self) -> bool: def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: """Write one instruction, sans prologue and epilogue.""" + # Write a static assertion that a family's cache size is correct - if family := self.family: - if self.name == family.name: - if cache_size := family.size: - out.emit( - f"static_assert({cache_size} == " - f'{self.cache_offset}, "incorrect cache size");' - ) + out.static_assert_family_size(self.name, self.family, self.cache_offset) # Write input stack effect variable declarations and initializations - ieffects = list(reversed(self.input_effects)) - for i, ieffect in enumerate(ieffects): - isize = string_effect_size( - list_effect_size([ieff for ieff in ieffects[: i + 1]]) - ) - if ieffect.size: - src = StackEffect( - f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **" - ) - elif ieffect.cond: - src = StackEffect( - f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", - "", - ) - else: - src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") - out.declare(ieffect, src) - - # Write output stack effect variable declarations - isize = string_effect_size(list_effect_size(self.input_effects)) - input_names = {ieffect.name for ieffect in self.input_effects} - for i, oeffect in enumerate(self.output_effects): - if oeffect.name not in input_names: - if oeffect.size: - osize = string_effect_size( - list_effect_size([oeff for oeff in self.output_effects[:i]]) - ) - offset = "stack_pointer" - if isize != osize: - if isize != "0": - offset += f" - ({isize})" - if osize != "0": - offset += f" + {osize}" - src = StackEffect(offset, "PyObject **") - out.declare(oeffect, src) - else: - out.declare(oeffect, None) - - # out.emit(f"next_instr += OPSIZE({self.inst.name}) - 1;") - - self.write_body(out, 0, self.active_caches, tier=tier) + stacking.write_single_instr(self, out, tier) # Skip the rest if the block always exits if self.always_exits: return - # Write net stack growth/shrinkage - out.stack_adjust( - [ieff for ieff in self.input_effects], - [oeff for oeff in self.output_effects], - ) - - # Write output stack effect assignments - oeffects = list(reversed(self.output_effects)) - for i, oeffect in enumerate(oeffects): - if oeffect.name in self.unmoved_names: - continue - osize = string_effect_size( - list_effect_size([oeff for oeff in oeffects[: i + 1]]) - ) - if oeffect.size: - dst = StackEffect( - f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **" - ) - else: - dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") - out.assign(dst, oeffect) - # Write cache effect if tier == TIER_ONE and self.cache_offset: out.emit(f"next_instr += {self.cache_offset};") @@ -274,7 +208,12 @@ def write_body( # These aren't DECREF'ed so they can stay. ieffs = list(self.input_effects) oeffs = list(self.output_effects) - while ieffs and oeffs and ieffs[0] == oeffs[0]: + while ( + ieffs + and oeffs + and ieffs[0] == oeffs[0] + and ieffs[0].name == oeffs[0].name + ): ieffs.pop(0) oeffs.pop(0) ninputs, symbolic = list_effect_size(ieffs) @@ -307,30 +246,13 @@ def write_body( InstructionOrCacheEffect = Instruction | parsing.CacheEffect -StackEffectMapping = list[tuple[StackEffect, StackEffect]] @dataclasses.dataclass class Component: instr: Instruction - input_mapping: StackEffectMapping - output_mapping: StackEffectMapping active_caches: list[ActiveCacheEffect] - def write_body(self, out: Formatter) -> None: - with out.block(""): - input_names = {ieffect.name for _, ieffect in self.input_mapping} - for var, ieffect in self.input_mapping: - out.declare(ieffect, var) - for _, oeffect in self.output_mapping: - if oeffect.name not in input_names: - out.declare(oeffect, None) - - self.instr.write_body(out, -4, self.active_caches) - - for var, oeffect in self.output_mapping: - out.assign(var, oeffect) - MacroParts = list[Component | parsing.CacheEffect] @@ -340,9 +262,6 @@ class MacroInstruction: """A macro instruction.""" name: str - stack: list[StackEffect] - initial_sp: int - final_sp: int instr_fmt: str instr_flags: InstructionFlags macro: parsing.Macro diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 290285dc03123c..5610ac661a8945 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -69,12 +69,18 @@ class Block(Node): @dataclass class StackEffect(Node): - name: str + name: str = field(compare=False) # __eq__ only uses type, cond, size type: str = "" # Optional `:type` cond: str = "" # Optional `if (cond)` size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond + def __repr__(self): + items = [self.name, self.type, self.cond, self.size] + while items and items[-1] == "": + del items[-1] + return f"StackEffect({', '.join(repr(item) for item in items)})" + @dataclass class Expression(Node): @@ -130,6 +136,7 @@ class Family(Node): size: str # Variable giving the cache size in code units members: list[str] + @dataclass class Pseudo(Node): name: str @@ -154,7 +161,13 @@ def inst_def(self) -> InstDef | None: if hdr := self.inst_header(): if block := self.block(): return InstDef( - hdr.override, hdr.register, hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block + hdr.override, + hdr.register, + hdr.kind, + hdr.name, + hdr.inputs, + hdr.outputs, + block, ) raise self.make_syntax_error("Expected block") return None @@ -371,9 +384,7 @@ def pseudo_def(self) -> Pseudo | None: raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo( - tkn.text, members - ) + return Pseudo(tkn.text, members) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/plexer.py b/Tools/cases_generator/plexer.py index a73254ed5b1daa..cb6c5375866490 100644 --- a/Tools/cases_generator/plexer.py +++ b/Tools/cases_generator/plexer.py @@ -1,4 +1,5 @@ import lexer as lx + Token = lx.Token @@ -64,7 +65,9 @@ def require(self, kind: str) -> Token: tkn = self.next() if tkn is not None and tkn.kind == kind: return tkn - raise self.make_syntax_error(f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn) + raise self.make_syntax_error( + f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn + ) def extract_line(self, lineno: int) -> str: # Return source line `lineno` (1-based) @@ -73,18 +76,20 @@ def extract_line(self, lineno: int) -> str: return "" return lines[lineno - 1] - def make_syntax_error(self, message: str, tkn: Token|None = None) -> SyntaxError: + def make_syntax_error(self, message: str, tkn: Token | None = None) -> SyntaxError: # Construct a SyntaxError instance from message and token if tkn is None: tkn = self.peek() if tkn is None: tkn = self.tokens[-1] - return lx.make_syntax_error(message, - self.filename, tkn.line, tkn.column, self.extract_line(tkn.line)) + return lx.make_syntax_error( + message, self.filename, tkn.line, tkn.column, self.extract_line(tkn.line) + ) if __name__ == "__main__": import sys + if sys.argv[1:]: filename = sys.argv[1] if filename == "-c" and sys.argv[2:]: diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py new file mode 100644 index 00000000000000..23eca3037f896d --- /dev/null +++ b/Tools/cases_generator/stacking.py @@ -0,0 +1,400 @@ +import dataclasses +import typing + +from formatting import ( + Formatter, + UNUSED, + maybe_parenthesize, + parenthesize_cond, +) +from instructions import ( + ActiveCacheEffect, + Instruction, + MacroInstruction, + Component, + Tiers, + TIER_ONE, +) +from parsing import StackEffect, CacheEffect, Family + + +@dataclasses.dataclass +class StackOffset: + """Represent the stack offset for a PEEK or POKE. + + - At stack_pointer[0], deep and high are both empty. + (Note that that is an invalid stack reference.) + - Below stack top, only deep is non-empty. + - Above stack top, only high is non-empty. + - In complex cases, both deep and high may be non-empty. + + All this would be much simpler if all stack entries were the same + size, but with conditional and array effects, they aren't. + The offsets are each represented by a list of StackEffect objects. + The name in the StackEffects is unused. + """ + + deep: list[StackEffect] = dataclasses.field(default_factory=list) + high: list[StackEffect] = dataclasses.field(default_factory=list) + + def clone(self) -> "StackOffset": + return StackOffset(list(self.deep), list(self.high)) + + def negate(self) -> "StackOffset": + return StackOffset(list(self.high), list(self.deep)) + + def deeper(self, eff: StackEffect) -> None: + if eff in self.high: + self.high.remove(eff) + else: + self.deep.append(eff) + + def higher(self, eff: StackEffect) -> None: + if eff in self.deep: + self.deep.remove(eff) + else: + self.high.append(eff) + + def as_terms(self) -> list[tuple[str, str]]: + num = 0 + terms: list[tuple[str, str]] = [] + for eff in self.deep: + if eff.size: + terms.append(("-", maybe_parenthesize(eff.size))) + elif eff.cond and eff.cond != "1": + terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) + elif eff.cond != "0": + num -= 1 + for eff in self.high: + if eff.size: + terms.append(("+", maybe_parenthesize(eff.size))) + elif eff.cond and eff.cond != "1": + terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) + elif eff.cond != "0": + num += 1 + if num < 0: + terms.insert(0, ("-", str(-num))) + elif num > 0: + terms.append(("+", str(num))) + return terms + + def as_index(self) -> str: + terms = self.as_terms() + return make_index(terms) + + +def make_index(terms: list[tuple[str, str]]) -> str: + # Produce an index expression from the terms honoring PEP 8, + # surrounding binary ops with spaces but not unary minus + index = "" + for sign, term in terms: + if index: + index += f" {sign} {term}" + elif sign == "+": + index = term + else: + index = sign + term + return index or "0" + + +@dataclasses.dataclass +class StackItem: + offset: StackOffset + effect: StackEffect + + def as_variable(self, lax: bool = False) -> str: + """Return e.g. stack_pointer[-1].""" + terms = self.offset.as_terms() + if self.effect.size: + terms.insert(0, ("+", "stack_pointer")) + index = make_index(terms) + if self.effect.size: + res = index + else: + res = f"stack_pointer[{index}]" + if not lax: + # Check that we're not reading or writing above stack top. + # Skip this for output variable initialization (lax=True). + assert ( + self.effect in self.offset.deep and not self.offset.high + ), f"Push or pop above current stack level: {res}" + return res + + +@dataclasses.dataclass +class CopyEffect: + src: StackEffect + dst: StackEffect + + +class EffectManager: + """Manage stack effects and offsets for an instruction.""" + + instr: Instruction + active_caches: list[ActiveCacheEffect] + peeks: list[StackItem] + pokes: list[StackItem] + copies: list[CopyEffect] # See merge() + # Track offsets from stack pointer + min_offset: StackOffset + final_offset: StackOffset + + def __init__( + self, + instr: Instruction, + active_caches: list[ActiveCacheEffect], + pred: "EffectManager | None" = None, + ): + self.instr = instr + self.active_caches = active_caches + self.peeks = [] + self.pokes = [] + self.copies = [] + self.final_offset = pred.final_offset.clone() if pred else StackOffset() + for eff in reversed(instr.input_effects): + self.final_offset.deeper(eff) + self.peeks.append(StackItem(offset=self.final_offset.clone(), effect=eff)) + self.min_offset = self.final_offset.clone() + for eff in instr.output_effects: + self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) + self.final_offset.higher(eff) + + if pred: + # Replace push(x) + pop(y) with copy(x, y). + # Check that the sources and destinations are disjoint. + sources: set[str] = set() + destinations: set[str] = set() + while ( + pred.pokes + and self.peeks + and pred.pokes[-1].effect == self.peeks[-1].effect + ): + src = pred.pokes.pop(-1).effect + dst = self.peeks.pop(0).effect + pred.final_offset.deeper(src) + if dst.name != UNUSED: + destinations.add(dst.name) + if dst.name != src.name: + sources.add(src.name) + self.copies.append(CopyEffect(src, dst)) + # TODO: Turn this into an error (pass an Analyzer instance?) + assert sources & destinations == set(), ( + pred.instr.name, + self.instr.name, + sources, + destinations, + ) + + def adjust_deeper(self, eff: StackEffect) -> None: + for peek in self.peeks: + peek.offset.deeper(eff) + for poke in self.pokes: + poke.offset.deeper(eff) + self.min_offset.deeper(eff) + self.final_offset.deeper(eff) + + def adjust_higher(self, eff: StackEffect) -> None: + for peek in self.peeks: + peek.offset.higher(eff) + for poke in self.pokes: + poke.offset.higher(eff) + self.min_offset.higher(eff) + self.final_offset.higher(eff) + + def adjust(self, offset: StackOffset) -> None: + for down in offset.deep: + self.adjust_deeper(down) + for up in offset.high: + self.adjust_higher(up) + + def adjust_inverse(self, offset: StackOffset) -> None: + for down in offset.deep: + self.adjust_higher(down) + for up in offset.high: + self.adjust_deeper(up) + + def collect_vars(self) -> dict[str, StackEffect]: + """Collect all variables, skipping unused ones.""" + vars: dict[str, StackEffect] = {} + + def add(eff: StackEffect) -> None: + if eff.name != UNUSED: + if eff.name in vars: + # TODO: Make this an error + assert vars[eff.name] == eff, ( + self.instr.name, + eff.name, + vars[eff.name], + eff, + ) + else: + vars[eff.name] = eff + + for copy in self.copies: + add(copy.src) + add(copy.dst) + for peek in self.peeks: + add(peek.effect) + for poke in self.pokes: + add(poke.effect) + + return vars + + +def less_than(a: StackOffset, b: StackOffset) -> bool: + # TODO: Handle more cases + if a.high != b.high: + return False + return a.deep[: len(b.deep)] == b.deep + + +def get_managers(parts: list[Component]) -> list[EffectManager]: + managers: list[EffectManager] = [] + pred: EffectManager | None = None + for part in parts: + mgr = EffectManager(part.instr, part.active_caches, pred) + managers.append(mgr) + pred = mgr + return managers + + +def get_stack_effect_info_for_macro(mac: MacroInstruction) -> tuple[str, str]: + """Get the stack effect info for a macro instruction. + + Returns a tuple (popped, pushed) where each is a string giving a + symbolic expression for the number of values popped/pushed. + """ + parts = [part for part in mac.parts if isinstance(part, Component)] + managers = get_managers(parts) + popped = StackOffset() + for mgr in managers: + if less_than(mgr.min_offset, popped): + popped = mgr.min_offset.clone() + # Compute pushed = final - popped + pushed = managers[-1].final_offset.clone() + for effect in popped.deep: + pushed.higher(effect) + for effect in popped.high: + pushed.deeper(effect) + return popped.negate().as_index(), pushed.as_index() + + +def write_single_instr( + instr: Instruction, out: Formatter, tier: Tiers = TIER_ONE +) -> None: + try: + write_components( + [Component(instr, instr.active_caches)], + out, + tier, + ) + except AssertionError as err: + raise AssertionError(f"Error writing instruction {instr.name}") from err + + +def write_macro_instr( + mac: MacroInstruction, out: Formatter, family: Family | None +) -> None: + parts = [part for part in mac.parts if isinstance(part, Component)] + + cache_adjust = 0 + for part in mac.parts: + match part: + case CacheEffect(size=size): + cache_adjust += size + case Component(instr=instr): + cache_adjust += instr.cache_offset + case _: + typing.assert_never(part) + + out.emit("") + with out.block(f"TARGET({mac.name})"): + if mac.predicted: + out.emit(f"PREDICTED({mac.name});") + out.static_assert_family_size(mac.name, family, cache_adjust) + try: + write_components(parts, out, TIER_ONE) + except AssertionError as err: + raise AssertionError(f"Error writing macro {mac.name}") from err + if cache_adjust: + out.emit(f"next_instr += {cache_adjust};") + out.emit("DISPATCH();") + + +def write_components( + parts: list[Component], + out: Formatter, + tier: Tiers, +) -> None: + managers = get_managers(parts) + + all_vars: dict[str, StackEffect] = {} + for mgr in managers: + for name, eff in mgr.collect_vars().items(): + if name in all_vars: + # TODO: Turn this into an error -- variable conflict + assert all_vars[name] == eff, ( + name, + mgr.instr.name, + all_vars[name], + eff, + ) + else: + all_vars[name] = eff + + # Declare all variables + for name, eff in all_vars.items(): + out.declare(eff, None) + + for mgr in managers: + if len(parts) > 1: + out.emit(f"// {mgr.instr.name}") + + for copy in mgr.copies: + if copy.src.name != copy.dst.name: + out.assign(copy.dst, copy.src) + for peek in mgr.peeks: + out.assign( + peek.effect, + StackEffect( + peek.as_variable(), + peek.effect.type, + peek.effect.cond, + peek.effect.size, + ), + ) + # Initialize array outputs + for poke in mgr.pokes: + if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + poke.effect, + StackEffect( + poke.as_variable(lax=True), + poke.effect.type, + poke.effect.cond, + poke.effect.size, + ), + ) + + if len(parts) == 1: + mgr.instr.write_body(out, 0, mgr.active_caches, tier) + else: + with out.block(""): + mgr.instr.write_body(out, -4, mgr.active_caches, tier) + + if mgr is managers[-1]: + out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + + for poke in mgr.pokes: + if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + StackEffect( + poke.as_variable(), + poke.effect.type, + poke.effect.cond, + poke.effect.size, + ), + poke.effect, + ) From 321f0f79325adfe7656645060c2008d5779e1a7f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 4 Aug 2023 19:49:07 +0200 Subject: [PATCH 029/598] gh-104683: Argument Clinic: Use CConverter.length_name where possible (#107638) Also make it a cached property. Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6eb2c550e696fc..917f1bfeb1d250 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2779,7 +2779,7 @@ class CConverter(metaclass=CConverterAutoRegister): # Only used by the 'O!' format unit (and the "object" converter). subclass_of: str | None = None - # Do we want an adjacent '_length' variable for this variable? + # See also the 'length_name' property. # Only used by format units ending with '#'. length = False @@ -2873,12 +2873,12 @@ def _render_self(self, parameter: Parameter, data: CRenderData) -> None: s = ("&" if self.impl_by_reference else "") + name data.impl_arguments.append(s) if self.length: - data.impl_arguments.append(self.length_name()) + data.impl_arguments.append(self.length_name) # impl_parameters data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) if self.length: - data.impl_parameters.append("Py_ssize_t " + self.length_name()) + data.impl_parameters.append(f"Py_ssize_t {self.length_name}") def _render_non_self( self, @@ -2937,6 +2937,7 @@ def render(self, parameter: Parameter, data: CRenderData) -> None: self._render_self(parameter, data) self._render_non_self(parameter, data) + @functools.cached_property def length_name(self) -> str: """Computes the name of the associated "length" variable.""" assert self.length is not None @@ -2960,7 +2961,7 @@ def parse_argument(self, args: list[str]) -> None: args.append(s) if self.length: - args.append("&" + self.length_name()) + args.append(f"&{self.length_name}") # # All the functions after here are intended as extension points. @@ -3005,9 +3006,8 @@ def declaration(self, *, in_parser: bool = False) -> str: declaration.append(default) declaration.append(";") if self.length: - declaration.append('\nPy_ssize_t ') - declaration.append(self.length_name()) - declaration.append(';') + declaration.append('\n') + declaration.append(f"Py_ssize_t {self.length_name};") return "".join(declaration) def initialize(self) -> str: @@ -3686,29 +3686,29 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); goto exit; }}}} - Py_ssize_t {paramname}_length; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ goto exit; }}}} - if (strlen({paramname}) != (size_t){paramname}_length) {{{{ + if (strlen({paramname}) != (size_t){length_name}) {{{{ PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + displayname=displayname, length_name=self.length_name) if self.format_unit == 'z': return """ if ({argname} == Py_None) {{{{ {paramname} = NULL; }}}} else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {paramname}_length; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ goto exit; }}}} - if (strlen({paramname}) != (size_t){paramname}_length) {{{{ + if (strlen({paramname}) != (size_t){length_name}) {{{{ PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} @@ -3718,7 +3718,7 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: goto exit; }}}} """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + displayname=displayname, length_name=self.length_name) return super().parse_arg(argname, displayname) # From 904b5319b3cc72063f4bfcd7beb3a1ef0fc641be Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 4 Aug 2023 21:33:34 +0300 Subject: [PATCH 030/598] Docs: Only include Plausible for html, not for epub etc (#107637) Only include Plausible for html, not for epub etc --- Doc/tools/templates/layout.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9832feba141675..80103158ea01e6 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,7 +26,9 @@ {% endblock %} {% block extrahead %} - + {% if builder == "html" %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From 2c25bd82f46df72c89ca5bca10eaa06137ab8290 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 4 Aug 2023 20:41:04 +0100 Subject: [PATCH 031/598] gh-106368: Argument clinic: improve coverage for `self.valid_line()` calls (#107641) --- Lib/test/test_clinic.py | 32 ++++++++++++++++++++++++++++++++ Tools/clinic/clinic.py | 4 +--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index dd17d02519bfa7..84b6a193ecf914 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1057,6 +1057,38 @@ def test_explicit_parameters_in_docstring(self): Okay, we're done here. """) + def test_docstring_with_comments(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + # We're about to have + # the documentation for x. + Documentation for x. + # We've just had + # the documentation for x. + y: int + + # We're about to have + # the documentation for foo. + This is the documentation for foo. + # We've just had + # the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + def test_parser_regression_special_character_in_parameter_column_of_docstring_first_line(self): function = self.parse_function(dedent(""" module os diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 917f1bfeb1d250..7fbae1e0d870ad 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4685,9 +4685,7 @@ def state_modulename_name(self, line: str) -> None: # this line is permitted to start with whitespace. # we'll call this number of spaces F (for "function"). - if not self.valid_line(line): - return - + assert self.valid_line(line) self.indent.infer(line) # are we cloning? From ec0a0d2bd9faa247d5b3208a8138e4399b2da890 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 5 Aug 2023 00:12:12 +0100 Subject: [PATCH 032/598] GH-70303: Emit FutureWarning when pathlib glob pattern ends with `**` (GH-105413) In a future Python release, patterns with this ending will match both files and directories. Users may add a trailing slash to remove the warning. --- Doc/library/pathlib.rst | 5 +++++ Lib/pathlib.py | 5 +++++ Lib/test/test_pathlib.py | 19 ++++++++++++++++--- ...3-06-07-00-13-00.gh-issue-70303.frwUKH.rst | 4 ++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 01dabe286969bb..22360b22fd924b 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -976,6 +976,11 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.13 The *follow_symlinks* parameter was added. + .. versionchanged:: 3.13 + Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a + future Python release, patterns with this ending will match both files + and directories. Add a trailing slash to match only directories. + .. method:: Path.group() Return the name of the group owning the file. :exc:`KeyError` is raised diff --git a/Lib/pathlib.py b/Lib/pathlib.py index c83cf3d2ef696d..758f70f5ba7077 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1069,6 +1069,11 @@ def _glob(self, pattern, case_sensitive, follow_symlinks): pattern_parts.append('') if pattern_parts[-1] == '**': # GH-70303: '**' only matches directories. Add trailing slash. + warnings.warn( + "Pattern ending '**' will match files and directories in a " + "future Python release. Add a trailing slash to match only " + "directories and remove this warning.", + FutureWarning, 3) pattern_parts.append('') if case_sensitive is None: diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 5789a932c59037..74deec84336f72 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1903,11 +1903,11 @@ def _check(glob, expected): "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**"), ["dirC/dirD"]) + _check(p.rglob("dir*/**/"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) - _check(p.rglob("**"), ["dirC", "dirC/dirD"]) + _check(p.rglob("**/"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) @@ -2057,7 +2057,20 @@ def test_glob_above_recursion_limit(self): path.mkdir(parents=True) with set_recursion_limit(recursion_limit): - list(base.glob('**')) + list(base.glob('**/')) + + def test_glob_recursive_no_trailing_slash(self): + P = self.cls + p = P(BASE) + with self.assertWarns(FutureWarning): + p.glob('**') + with self.assertWarns(FutureWarning): + p.glob('*/**') + with self.assertWarns(FutureWarning): + p.rglob('**') + with self.assertWarns(FutureWarning): + p.rglob('*/**') + def test_readlink(self): if not self.can_symlink: diff --git a/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst b/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst new file mode 100644 index 00000000000000..39a891ac5964ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst @@ -0,0 +1,4 @@ +Emit :exc:`FutureWarning` from :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob` if the given pattern ends with "``**``". In a +future Python release, patterns with this ending will match both files and +directories. Add a trailing slash to only match directories. From 05a824f294f1409f33e32f1799d5b413dcf24445 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 4 Aug 2023 16:24:50 -0700 Subject: [PATCH 033/598] GH-84436: Skip refcounting for known immortals (GH-107605) --- Include/internal/pycore_long.h | 6 ++-- ...3-08-03-13-38-14.gh-issue-84436.gl1wHx.rst | 1 + Modules/_asynciomodule.c | 3 +- Modules/_io/textio.c | 6 ++-- Modules/_json.c | 6 ++-- Modules/_pickle.c | 3 +- Objects/boolobject.c | 3 +- Objects/bytesobject.c | 27 +++++++-------- Objects/funcobject.c | 4 +-- Objects/longobject.c | 5 ++- Objects/rangeobject.c | 4 +-- Objects/sliceobject.c | 4 +-- Objects/tupleobject.c | 2 +- Objects/typeobject.c | 2 +- Objects/unicodeobject.c | 33 +++++++------------ Python/ceval.c | 2 +- Python/context.c | 2 +- Python/fileutils.c | 2 +- Python/hamt.c | 2 +- 19 files changed, 52 insertions(+), 65 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3f8d8adc83b20a..3dc00ec7e04c6f 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,19 +64,19 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -// Return a borrowed reference to the zero singleton. +// Return a reference to the immortal zero singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetZero(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS]; } -// Return a borrowed reference to the one singleton. +// Return a reference to the immortal one singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetOne(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+1]; } static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) { - return Py_NewRef((PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]); + return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]; } extern PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst new file mode 100644 index 00000000000000..71044c32feebcc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst @@ -0,0 +1 @@ +Skip reference count modifications for many known immortal objects. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index f5a589b00c48d0..39c803355ba95b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1398,7 +1398,8 @@ FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) default: assert (0); } - return Py_XNewRef(ret); + assert(_Py_IsImmortal(ret)); + return ret; } static PyObject * diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 24d846e6634375..6ce90b2ed774c0 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -234,7 +234,7 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, { if (errors == NULL) { - errors = Py_NewRef(&_Py_ID(strict)); + errors = &_Py_ID(strict); } else { errors = Py_NewRef(errors); @@ -1138,7 +1138,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, if (encoding == NULL && _PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); - self->encoding = Py_NewRef(&_Py_STR(utf_8)); + self->encoding = &_Py_STR(utf_8); } else if (encoding == NULL || (strcmp(encoding, "locale") == 0)) { self->encoding = _Py_GetLocaleEncodingObject(); @@ -2267,7 +2267,7 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) Py_CLEAR(chunks); } if (line == NULL) { - line = Py_NewRef(&_Py_STR(empty)); + line = &_Py_STR(empty); } return line; diff --git a/Modules/_json.c b/Modules/_json.c index 4fcaa07d9cfd81..c7cfe50b52faff 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1277,13 +1277,13 @@ _encoded_const(PyObject *obj) { /* Return the JSON string representation of None, True, False */ if (obj == Py_None) { - return Py_NewRef(&_Py_ID(null)); + return &_Py_ID(null); } else if (obj == Py_True) { - return Py_NewRef(&_Py_ID(true)); + return &_Py_ID(true); } else if (obj == Py_False) { - return Py_NewRef(&_Py_ID(false)); + return &_Py_ID(false); } else { PyErr_SetString(PyExc_ValueError, "not a const"); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d4c0be78935235..c2b04cc513a664 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2029,8 +2029,7 @@ whichmodule(PyObject *global, PyObject *dotted_path) } /* If no module is found, use __main__. */ - module_name = &_Py_ID(__main__); - return Py_NewRef(module_name); + return &_Py_ID(__main__); } /* fast_save_enter() and fast_save_leave() are guards against recursive diff --git a/Objects/boolobject.c b/Objects/boolobject.c index bbb187cb7121e7..e2e359437f0edf 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -13,8 +13,7 @@ static PyObject * bool_repr(PyObject *self) { - PyObject *res = self == Py_True ? &_Py_ID(True) : &_Py_ID(False); - return Py_NewRef(res); + return self == Py_True ? &_Py_ID(True) : &_Py_ID(False); } /* Function to return a bool from a C long */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 6b9231a9fa7693..afe9192720c628 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -41,17 +41,12 @@ Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, #define EMPTY (&_Py_SINGLETON(bytes_empty)) -// Return a borrowed reference to the empty bytes string singleton. +// Return a reference to the immortal empty bytes string singleton. static inline PyObject* bytes_get_empty(void) { - return &EMPTY->ob_base.ob_base; -} - - -// Return a strong reference to the empty bytes string singleton. -static inline PyObject* bytes_new_empty(void) -{ - return Py_NewRef(EMPTY); + PyObject *empty = &EMPTY->ob_base.ob_base; + assert(_Py_IsImmortal(empty)); + return empty; } @@ -84,7 +79,7 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) assert(size >= 0); if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } if ((size_t)size > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { @@ -123,10 +118,11 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) } if (size == 1 && str != NULL) { op = CHARACTER(*str & 255); - return Py_NewRef(op); + assert(_Py_IsImmortal(op)); + return (PyObject *)op; } if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } op = (PyBytesObject *)_PyBytes_FromSize(size, 0); @@ -154,11 +150,12 @@ PyBytes_FromString(const char *str) } if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } else if (size == 1) { op = CHARACTER(*str & 255); - return Py_NewRef(op); + assert(_Py_IsImmortal(op)); + return (PyObject *)op; } /* Inline PyObject_NewVar */ @@ -3065,7 +3062,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) goto error; } if (newsize == 0) { - *pv = bytes_new_empty(); + *pv = bytes_get_empty(); Py_DECREF(v); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 7fffa1c8bbff96..8c0bface3ac710 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -831,8 +831,8 @@ func_clear(PyFunctionObject *op) // However, name and qualname could be str subclasses, so they // could have reference cycles. The solution is to replace them // with a genuinely immutable string. - Py_SETREF(op->func_name, Py_NewRef(&_Py_STR(empty))); - Py_SETREF(op->func_qualname, Py_NewRef(&_Py_STR(empty))); + Py_SETREF(op->func_name, &_Py_STR(empty)); + Py_SETREF(op->func_qualname, &_Py_STR(empty)); return 0; } diff --git a/Objects/longobject.c b/Objects/longobject.c index 5d9b413861478a..354cba9d6d800f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -174,7 +174,7 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) { assert(digit_count >= 0); if (digit_count == 0) { - return (PyLongObject *)Py_NewRef(_PyLong_GetZero()); + return (PyLongObject *)_PyLong_GetZero(); } PyLongObject *result = _PyLong_New(digit_count); if (result == NULL) { @@ -2857,8 +2857,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, if (*prem == NULL) { return -1; } - PyObject *zero = _PyLong_GetZero(); - *pdiv = (PyLongObject*)Py_NewRef(zero); + *pdiv = (PyLongObject*)_PyLong_GetZero(); return 0; } if (size_b == 1) { diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1e3d5acc8ae6f8..6e06bef95032cf 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -106,8 +106,8 @@ range_from_array(PyTypeObject *type, PyObject *const *args, Py_ssize_t num_args) if (!stop) { return NULL; } - start = Py_NewRef(_PyLong_GetZero()); - step = Py_NewRef(_PyLong_GetOne()); + start = _PyLong_GetZero(); + step = _PyLong_GetOne(); break; case 0: PyErr_SetString(PyExc_TypeError, diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index dc3aad11ce10e5..8cf654fb6f812d 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -415,7 +415,7 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, /* Convert step to an integer; raise for zero step. */ if (self->step == Py_None) { - step = Py_NewRef(_PyLong_GetOne()); + step = _PyLong_GetOne(); step_is_negative = 0; } else { @@ -443,7 +443,7 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, goto error; } else { - lower = Py_NewRef(_PyLong_GetZero()); + lower = _PyLong_GetZero(); upper = Py_NewRef(length); } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c3ff40fdb14c60..b669a3dd8525eb 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -62,7 +62,7 @@ tuple_alloc(Py_ssize_t size) static inline PyObject * tuple_get_empty(void) { - return Py_NewRef(&_Py_SINGLETON(tuple_empty)); + return (PyObject *)&_Py_SINGLETON(tuple_empty); } PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index abe33f1562059d..76809494dd8817 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1085,7 +1085,7 @@ type_module(PyTypeObject *type, void *context) PyUnicode_InternInPlace(&mod); } else { - mod = Py_NewRef(&_Py_ID(builtins)); + mod = &_Py_ID(builtins); } } return mod; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index cc979b2ef28d76..c6876d4ca0ef0f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -211,21 +211,13 @@ static int unicode_is_singleton(PyObject *unicode); #endif -// Return a borrowed reference to the empty string singleton. +// Return a reference to the immortal empty string singleton. static inline PyObject* unicode_get_empty(void) { _Py_DECLARE_STR(empty, ""); return &_Py_STR(empty); } - -// Return a strong reference to the empty string singleton. -static inline PyObject* unicode_new_empty(void) -{ - PyObject *empty = unicode_get_empty(); - return Py_NewRef(empty); -} - /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -310,7 +302,7 @@ clear_interned_dict(PyInterpreterState *interp) #define _Py_RETURN_UNICODE_EMPTY() \ do { \ - return unicode_new_empty(); \ + return unicode_get_empty(); \ } while (0) static inline void @@ -650,7 +642,6 @@ unicode_result(PyObject *unicode) PyObject *empty = unicode_get_empty(); if (unicode != empty) { Py_DECREF(unicode); - Py_INCREF(empty); } return empty; } @@ -662,7 +653,6 @@ unicode_result(PyObject *unicode) Py_UCS1 ch = data[0]; PyObject *latin1_char = LATIN1(ch); if (unicode != latin1_char) { - Py_INCREF(latin1_char); Py_DECREF(unicode); } return latin1_char; @@ -1199,7 +1189,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) { /* Optimization for empty strings */ if (size == 0) { - return unicode_new_empty(); + return unicode_get_empty(); } PyObject *obj; @@ -1669,7 +1659,7 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length) return 0; if (length == 0) { - PyObject *empty = unicode_new_empty(); + PyObject *empty = unicode_get_empty(); Py_SETREF(*p_unicode, empty); return 0; } @@ -1764,7 +1754,9 @@ unicode_write_cstr(PyObject *unicode, Py_ssize_t index, static PyObject* get_latin1_char(Py_UCS1 ch) { - return Py_NewRef(LATIN1(ch)); + PyObject *o = LATIN1(ch); + assert(_Py_IsImmortal(o)); + return o; } static PyObject* @@ -1891,7 +1883,7 @@ PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) "NULL string with positive size with NULL passed to PyUnicode_FromStringAndSize"); return NULL; } - return unicode_new_empty(); + return unicode_get_empty(); } PyObject * @@ -10261,7 +10253,7 @@ replace(PyObject *self, PyObject *str1, } new_size = slen + n * (len2 - len1); if (new_size == 0) { - u = unicode_new_empty(); + u = unicode_get_empty(); goto done; } if (new_size > (PY_SSIZE_T_MAX / rkind)) { @@ -14505,7 +14497,7 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, { PyObject *unicode; if (x == NULL) { - unicode = unicode_new_empty(); + unicode = unicode_get_empty(); } else if (encoding == NULL && errors == NULL) { unicode = PyObject_Str(x); @@ -14994,8 +14986,7 @@ unicode_ascii_iter_next(unicodeiterobject *it) Py_UCS1 chr = (Py_UCS1)PyUnicode_READ(PyUnicode_1BYTE_KIND, data, it->it_index); it->it_index++; - PyObject *item = (PyObject*)&_Py_SINGLETON(strings).ascii[chr]; - return Py_NewRef(item); + return (PyObject*)&_Py_SINGLETON(strings).ascii[chr]; } it->it_seq = NULL; Py_DECREF(seq); @@ -15025,7 +15016,7 @@ unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored)) if (it->it_seq != NULL) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { - PyObject *u = unicode_new_empty(); + PyObject *u = unicode_get_empty(); if (u == NULL) { Py_XDECREF(iter); return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index b85e9677747afb..30f722e45e476b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1265,7 +1265,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_flags & CO_VARARGS) { PyObject *u = NULL; if (argcount == n) { - u = Py_NewRef(&_Py_SINGLETON(tuple_empty)); + u = (PyObject *)&_Py_SINGLETON(tuple_empty); } else { assert(args != NULL); diff --git a/Python/context.c b/Python/context.c index 9ac51874fb5be5..c94c014219d0e4 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1267,7 +1267,7 @@ PyTypeObject _PyContextTokenMissing_Type = { static PyObject * get_token_missing(void) { - return Py_NewRef(&_Py_SINGLETON(context_token_missing)); + return (PyObject *)&_Py_SINGLETON(context_token_missing); } diff --git a/Python/fileutils.c b/Python/fileutils.c index f262c3e095c9ba..19b23f6bd18b30 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -105,7 +105,7 @@ _Py_device_encoding(int fd) #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); - return Py_NewRef(&_Py_STR(utf_8)); + return &_Py_STR(utf_8); } return _Py_GetLocaleEncodingObject(); #endif diff --git a/Python/hamt.c b/Python/hamt.c index c78b5a7fab94f0..24265edc2c3fd4 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -514,7 +514,7 @@ hamt_node_bitmap_new(Py_ssize_t size) /* Since bitmap nodes are immutable, we can cache the instance for size=0 and reuse it whenever we need an empty bitmap node. */ - return (PyHamtNode *)Py_NewRef(&_Py_SINGLETON(hamt_bitmap_node_empty)); + return (PyHamtNode *)&_Py_SINGLETON(hamt_bitmap_node_empty); } assert(size >= 0); From 4e6fac7fcc31fc6198fcddc612688b0a05ff7ae4 Mon Sep 17 00:00:00 2001 From: Ivin Lee <62840497+rdrf2838@users.noreply.github.com> Date: Sat, 5 Aug 2023 12:10:46 +0800 Subject: [PATCH 034/598] gh-106608: make uop trace variable length (#107531) Executors are now more like tuples. --- Include/cpython/optimizer.h | 2 +- Include/internal/pycore_uops.h | 2 +- ...23-08-01-09-41-36.gh-issue-106608.OFZogw.rst | 1 + Python/optimizer.c | 17 ++++------------- 4 files changed, 7 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 2260501bfd608e..da34ec1882a539 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -12,7 +12,7 @@ typedef struct { } _PyVMData; typedef struct _PyExecutorObject { - PyObject_HEAD + PyObject_VAR_HEAD /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index edb141cc79f752..57a5970353b360 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -18,7 +18,7 @@ typedef struct { typedef struct { _PyExecutorObject base; - _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; // TODO: variable length + _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; _PyInterpreterFrame *_PyUopExecute( diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst new file mode 100644 index 00000000000000..20d43a7c4f754a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst @@ -0,0 +1 @@ +Make ``_PyUOpExecutorObject`` variable length. diff --git a/Python/optimizer.c b/Python/optimizer.c index 238ab02d09faa7..d2ed8dfcd2cff2 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -320,13 +320,7 @@ uop_name(int index) { static Py_ssize_t uop_len(_PyUOpExecutorObject *self) { - int count = 0; - for (; count < _Py_UOP_MAX_TRACE_LENGTH; count++) { - if (self->trace[count].opcode == 0) { - break; - } - } - return count; + return Py_SIZE(self); } static PyObject * @@ -368,8 +362,8 @@ PySequenceMethods uop_as_sequence = { static PyTypeObject UOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = sizeof(_PyUOpExecutorObject), - .tp_itemsize = 0, + .tp_basicsize = sizeof(_PyUOpExecutorObject) - sizeof(_PyUOpInstruction), + .tp_itemsize = sizeof(_PyUOpInstruction), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, @@ -699,15 +693,12 @@ uop_optimize( return trace_length; } OBJECT_STAT_INC(optimization_traces_created); - _PyUOpExecutorObject *executor = PyObject_New(_PyUOpExecutorObject, &UOpExecutor_Type); + _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); if (executor == NULL) { return -1; } executor->base.execute = _PyUopExecute; memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction)); - if (trace_length < _Py_UOP_MAX_TRACE_LENGTH) { - executor->trace[trace_length].opcode = 0; // Sentinel - } *exec_ptr = (_PyExecutorObject *)executor; return 1; } From 85e5b1f5b806289744ef9a5a13dabfb23044f713 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 4 Aug 2023 21:50:36 -0700 Subject: [PATCH 035/598] gh-106812: Fix two tiny bugs in analysis.py (#107649) This fixes two tiny defects in analysis.py that I didn't catch on time in #107564: - `get_var_names` in `check_macro_consistency` should skip `UNUSED` names. - Fix an occurrence of `is UNUSED` (should be `==`). --- Tools/cases_generator/analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index bd8918a87ffe08..2db1cd01c19ae5 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -297,6 +297,8 @@ def check_macro_consistency(self, mac: MacroInstruction) -> None: def get_var_names(instr: Instruction) -> dict[str, StackEffect]: vars: dict[str, StackEffect] = {} for eff in instr.input_effects + instr.output_effects: + if eff.name == UNUSED: + continue if eff.name in vars: if vars[eff.name] != eff: self.error( @@ -335,7 +337,7 @@ def get_var_names(instr: Instruction) -> dict[str, StackEffect]: copies: list[tuple[StackEffect, StackEffect]] = [] while pushes and pops and pushes[-1] == pops[0]: src, dst = pushes.pop(), pops.pop(0) - if src.name == dst.name or dst.name is UNUSED: + if src.name == dst.name or dst.name == UNUSED: continue copies.append((src, dst)) reads = set(copy[0].name for copy in copies) From 5e2746d6e2fb0da29225ead7135f078c5f087b57 Mon Sep 17 00:00:00 2001 From: Daniele Procida Date: Sat, 5 Aug 2023 13:28:28 +0200 Subject: [PATCH 036/598] gh-107432 Fix incorrect indentation in annotations HOWTO (#107445) gh-107432 Fix incorrect indentation in annotations document Body text in https://docs.python.org/3/howto/annotations.html was indented throughout, and was being rendered in blockquote elements. --- Doc/howto/annotations.rst | 326 +++++++++++++++++++------------------- 1 file changed, 163 insertions(+), 163 deletions(-) diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 472069032d6509..1134686c947d66 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -32,201 +32,201 @@ Annotations Best Practices Accessing The Annotations Dict Of An Object In Python 3.10 And Newer ==================================================================== - Python 3.10 adds a new function to the standard library: - :func:`inspect.get_annotations`. In Python versions 3.10 - and newer, calling this function is the best practice for - accessing the annotations dict of any object that supports - annotations. This function can also "un-stringize" - stringized annotations for you. - - If for some reason :func:`inspect.get_annotations` isn't - viable for your use case, you may access the - ``__annotations__`` data member manually. Best practice - for this changed in Python 3.10 as well: as of Python 3.10, - ``o.__annotations__`` is guaranteed to *always* work - on Python functions, classes, and modules. If you're - certain the object you're examining is one of these three - *specific* objects, you may simply use ``o.__annotations__`` - to get at the object's annotations dict. - - However, other types of callables--for example, - callables created by :func:`functools.partial`--may - not have an ``__annotations__`` attribute defined. When - accessing the ``__annotations__`` of a possibly unknown - object, best practice in Python versions 3.10 and - newer is to call :func:`getattr` with three arguments, - for example ``getattr(o, '__annotations__', None)``. - - Before Python 3.10, accessing ``__annotations__`` on a class that - defines no annotations but that has a parent class with - annotations would return the parent's ``__annotations__``. - In Python 3.10 and newer, the child class's annotations - will be an empty dict instead. +Python 3.10 adds a new function to the standard library: +:func:`inspect.get_annotations`. In Python versions 3.10 +and newer, calling this function is the best practice for +accessing the annotations dict of any object that supports +annotations. This function can also "un-stringize" +stringized annotations for you. + +If for some reason :func:`inspect.get_annotations` isn't +viable for your use case, you may access the +``__annotations__`` data member manually. Best practice +for this changed in Python 3.10 as well: as of Python 3.10, +``o.__annotations__`` is guaranteed to *always* work +on Python functions, classes, and modules. If you're +certain the object you're examining is one of these three +*specific* objects, you may simply use ``o.__annotations__`` +to get at the object's annotations dict. + +However, other types of callables--for example, +callables created by :func:`functools.partial`--may +not have an ``__annotations__`` attribute defined. When +accessing the ``__annotations__`` of a possibly unknown +object, best practice in Python versions 3.10 and +newer is to call :func:`getattr` with three arguments, +for example ``getattr(o, '__annotations__', None)``. + +Before Python 3.10, accessing ``__annotations__`` on a class that +defines no annotations but that has a parent class with +annotations would return the parent's ``__annotations__``. +In Python 3.10 and newer, the child class's annotations +will be an empty dict instead. Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== - In Python 3.9 and older, accessing the annotations dict - of an object is much more complicated than in newer versions. - The problem is a design flaw in these older versions of Python, - specifically to do with class annotations. +In Python 3.9 and older, accessing the annotations dict +of an object is much more complicated than in newer versions. +The problem is a design flaw in these older versions of Python, +specifically to do with class annotations. - Best practice for accessing the annotations dict of other - objects--functions, other callables, and modules--is the same - as best practice for 3.10, assuming you aren't calling - :func:`inspect.get_annotations`: you should use three-argument - :func:`getattr` to access the object's ``__annotations__`` - attribute. +Best practice for accessing the annotations dict of other +objects--functions, other callables, and modules--is the same +as best practice for 3.10, assuming you aren't calling +:func:`inspect.get_annotations`: you should use three-argument +:func:`getattr` to access the object's ``__annotations__`` +attribute. - Unfortunately, this isn't best practice for classes. The problem - is that, since ``__annotations__`` is optional on classes, and - because classes can inherit attributes from their base classes, - accessing the ``__annotations__`` attribute of a class may - inadvertently return the annotations dict of a *base class.* - As an example:: +Unfortunately, this isn't best practice for classes. The problem +is that, since ``__annotations__`` is optional on classes, and +because classes can inherit attributes from their base classes, +accessing the ``__annotations__`` attribute of a class may +inadvertently return the annotations dict of a *base class.* +As an example:: - class Base: - a: int = 3 - b: str = 'abc' + class Base: + a: int = 3 + b: str = 'abc' - class Derived(Base): - pass + class Derived(Base): + pass - print(Derived.__annotations__) + print(Derived.__annotations__) - This will print the annotations dict from ``Base``, not - ``Derived``. +This will print the annotations dict from ``Base``, not +``Derived``. - Your code will have to have a separate code path if the object - you're examining is a class (``isinstance(o, type)``). - In that case, best practice relies on an implementation detail - of Python 3.9 and before: if a class has annotations defined, - they are stored in the class's ``__dict__`` dictionary. Since - the class may or may not have annotations defined, best practice - is to call the ``get`` method on the class dict. +Your code will have to have a separate code path if the object +you're examining is a class (``isinstance(o, type)``). +In that case, best practice relies on an implementation detail +of Python 3.9 and before: if a class has annotations defined, +they are stored in the class's ``__dict__`` dictionary. Since +the class may or may not have annotations defined, best practice +is to call the ``get`` method on the class dict. - To put it all together, here is some sample code that safely - accesses the ``__annotations__`` attribute on an arbitrary - object in Python 3.9 and before:: +To put it all together, here is some sample code that safely +accesses the ``__annotations__`` attribute on an arbitrary +object in Python 3.9 and before:: - if isinstance(o, type): - ann = o.__dict__.get('__annotations__', None) - else: - ann = getattr(o, '__annotations__', None) + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) - After running this code, ``ann`` should be either a - dictionary or ``None``. You're encouraged to double-check - the type of ``ann`` using :func:`isinstance` before further - examination. +After running this code, ``ann`` should be either a +dictionary or ``None``. You're encouraged to double-check +the type of ``ann`` using :func:`isinstance` before further +examination. - Note that some exotic or malformed type objects may not have - a ``__dict__`` attribute, so for extra safety you may also wish - to use :func:`getattr` to access ``__dict__``. +Note that some exotic or malformed type objects may not have +a ``__dict__`` attribute, so for extra safety you may also wish +to use :func:`getattr` to access ``__dict__``. Manually Un-Stringizing Stringized Annotations ============================================== - In situations where some annotations may be "stringized", - and you wish to evaluate those strings to produce the - Python values they represent, it really is best to - call :func:`inspect.get_annotations` to do this work - for you. - - If you're using Python 3.9 or older, or if for some reason - you can't use :func:`inspect.get_annotations`, you'll need - to duplicate its logic. You're encouraged to examine the - implementation of :func:`inspect.get_annotations` in the - current Python version and follow a similar approach. - - In a nutshell, if you wish to evaluate a stringized annotation - on an arbitrary object ``o``: - - * If ``o`` is a module, use ``o.__dict__`` as the - ``globals`` when calling :func:`eval`. - * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` - as the ``globals``, and ``dict(vars(o))`` as the ``locals``, - when calling :func:`eval`. - * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively - unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as - appropriate, until you have found the root unwrapped function. - * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. - - However, not all string values used as annotations can - be successfully turned into Python values by :func:`eval`. - String values could theoretically contain any valid string, - and in practice there are valid use cases for type hints that - require annotating with string values that specifically - *can't* be evaluated. For example: - - * :pep:`604` union types using ``|``, before support for this - was added to Python 3.10. - * Definitions that aren't needed at runtime, only imported - when :const:`typing.TYPE_CHECKING` is true. - - If :func:`eval` attempts to evaluate such values, it will - fail and raise an exception. So, when designing a library - API that works with annotations, it's recommended to only - attempt to evaluate string values when explicitly requested - to by the caller. +In situations where some annotations may be "stringized", +and you wish to evaluate those strings to produce the +Python values they represent, it really is best to +call :func:`inspect.get_annotations` to do this work +for you. + +If you're using Python 3.9 or older, or if for some reason +you can't use :func:`inspect.get_annotations`, you'll need +to duplicate its logic. You're encouraged to examine the +implementation of :func:`inspect.get_annotations` in the +current Python version and follow a similar approach. + +In a nutshell, if you wish to evaluate a stringized annotation +on an arbitrary object ``o``: + +* If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. +* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. +* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. +* If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + +However, not all string values used as annotations can +be successfully turned into Python values by :func:`eval`. +String values could theoretically contain any valid string, +and in practice there are valid use cases for type hints that +require annotating with string values that specifically +*can't* be evaluated. For example: + +* :pep:`604` union types using ``|``, before support for this + was added to Python 3.10. +* Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + +If :func:`eval` attempts to evaluate such values, it will +fail and raise an exception. So, when designing a library +API that works with annotations, it's recommended to only +attempt to evaluate string values when explicitly requested +to by the caller. Best Practices For ``__annotations__`` In Any Python Version ============================================================ - * You should avoid assigning to the ``__annotations__`` member - of objects directly. Let Python manage setting ``__annotations__``. +* You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. - * If you do assign directly to the ``__annotations__`` member - of an object, you should always set it to a ``dict`` object. +* If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. - * If you directly access the ``__annotations__`` member - of an object, you should ensure that it's a - dictionary before attempting to examine its contents. +* If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. - * You should avoid modifying ``__annotations__`` dicts. +* You should avoid modifying ``__annotations__`` dicts. - * You should avoid deleting the ``__annotations__`` attribute - of an object. +* You should avoid deleting the ``__annotations__`` attribute + of an object. ``__annotations__`` Quirks ========================== - In all versions of Python 3, function - objects lazy-create an annotations dict if no annotations - are defined on that object. You can delete the ``__annotations__`` - attribute using ``del fn.__annotations__``, but if you then - access ``fn.__annotations__`` the object will create a new empty dict - that it will store and return as its annotations. Deleting the - annotations on a function before it has lazily created its annotations - dict will throw an ``AttributeError``; using ``del fn.__annotations__`` - twice in a row is guaranteed to always throw an ``AttributeError``. - - Everything in the above paragraph also applies to class and module - objects in Python 3.10 and newer. - - In all versions of Python 3, you can set ``__annotations__`` - on a function object to ``None``. However, subsequently - accessing the annotations on that object using ``fn.__annotations__`` - will lazy-create an empty dictionary as per the first paragraph of - this section. This is *not* true of modules and classes, in any Python - version; those objects permit setting ``__annotations__`` to any - Python value, and will retain whatever value is set. - - If Python stringizes your annotations for you - (using ``from __future__ import annotations``), and you - specify a string as an annotation, the string will - itself be quoted. In effect the annotation is quoted - *twice.* For example:: - - from __future__ import annotations - def foo(a: "str"): pass - - print(foo.__annotations__) - - This prints ``{'a': "'str'"}``. This shouldn't really be considered - a "quirk"; it's mentioned here simply because it might be surprising. +In all versions of Python 3, function +objects lazy-create an annotations dict if no annotations +are defined on that object. You can delete the ``__annotations__`` +attribute using ``del fn.__annotations__``, but if you then +access ``fn.__annotations__`` the object will create a new empty dict +that it will store and return as its annotations. Deleting the +annotations on a function before it has lazily created its annotations +dict will throw an ``AttributeError``; using ``del fn.__annotations__`` +twice in a row is guaranteed to always throw an ``AttributeError``. + +Everything in the above paragraph also applies to class and module +objects in Python 3.10 and newer. + +In all versions of Python 3, you can set ``__annotations__`` +on a function object to ``None``. However, subsequently +accessing the annotations on that object using ``fn.__annotations__`` +will lazy-create an empty dictionary as per the first paragraph of +this section. This is *not* true of modules and classes, in any Python +version; those objects permit setting ``__annotations__`` to any +Python value, and will retain whatever value is set. + +If Python stringizes your annotations for you +(using ``from __future__ import annotations``), and you +specify a string as an annotation, the string will +itself be quoted. In effect the annotation is quoted +*twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + +This prints ``{'a': "'str'"}``. This shouldn't really be considered +a "quirk"; it's mentioned here simply because it might be surprising. From 41178e41995992bbe417f94bce158de93f9e3188 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 5 Aug 2023 17:48:15 +0530 Subject: [PATCH 037/598] GH-106684: raise `ResourceWarning` when `asyncio.StreamWriter` is not closed (#107650) --- Lib/asyncio/streams.py | 6 +++++ Lib/test/test_asyncio/test_streams.py | 23 +++++++++++++++++++ ...-08-05-05-10-41.gh-issue-106684.P9zRXb.rst | 1 + 3 files changed, 30 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index bf15f517e50dce..b7ad365709b19e 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -5,6 +5,7 @@ import collections import socket import sys +import warnings import weakref if hasattr(socket, 'AF_UNIX'): @@ -392,6 +393,11 @@ async def start_tls(self, sslcontext, *, self._transport = new_transport protocol._replace_writer(self) + def __del__(self, warnings=warnings): + if not self._transport.is_closing(): + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) + class StreamReader: diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 7f9dc621808358..9c92e75886c593 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1074,6 +1074,29 @@ def test_eof_feed_when_closing_writer(self): self.assertEqual(messages, []) + def test_unclosed_resource_warnings(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + with self.assertWarns(ResourceWarning): + del wr + gc.collect() + + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst new file mode 100644 index 00000000000000..02c52d714e9df7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst @@ -0,0 +1 @@ +Raise :exc:`ResourceWarning` when :class:`asyncio.StreamWriter` is not closed leading to memory leaks. Patch by Kumar Aditya. From 9ebc6ecbc336d7b17cd158d1a4522f832df3e6e2 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 5 Aug 2023 10:38:20 -0400 Subject: [PATCH 038/598] gh-107662: Switch 'any' and 'anext' in functions.rst (#107663) Order was reversed in index at top, not in body. --- Doc/library/functions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2688f43f9b4ffc..b271067ae639c5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -14,8 +14,8 @@ are always available. They are listed here in alphabetical order. | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | | | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | | | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | +| | :func:`anext` | | | | | | :func:`round` | +| | :func:`any` | | **F** | | **M** | | | | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | From 4a5b4221e381c541f3f73537b7b87580d100158b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 5 Aug 2023 22:19:37 +0200 Subject: [PATCH 039/598] Docs: Argument Clinic: Improve 'How to write a custom converter' (#107328) - Omit unneccesary wording and sentences - Don't mention implementation details (no digression, explanation) Co-authored-by: Ezio Melotti --- Doc/howto/clinic.rst | 50 +++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 9c9a4f45dd0f58..dcede13a03b4a5 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1343,35 +1343,37 @@ state. Example from the ``setattro`` slot method in See also :pep:`573`. +.. _clinic-howto-custom-converter: + How to write a custom converter ------------------------------- -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from :py:class:`!CConverter`. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit—parsing this parameter means calling +A converter is a Python class that inherits from :py:class:`!CConverter`. +The main purpose of a custom converter, is for parameters parsed with +the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass :py:meth:`!CConverter.__init__`. Instead, you should -write a :py:meth:`!converter_init` function. :py:meth:`!converter_init` -always accepts a *self* parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -:py:meth:`!converter_init`. +Your converter class should be named :samp:`{ConverterName}_converter`. +By following this convention, your converter class will be automatically +registered with Argument Clinic, with its *converter name* being the name of +your converter class with the ``_converter`` suffix stripped off. -There are some additional members of :py:class:`!CConverter` you may wish -to specify in your subclass. Here's the current list: +Instead of subclassing :py:meth:`!CConverter.__init__`, +write a :py:meth:`!converter_init` method. +Apart for the *self* parameter, all additional :py:meth:`!converter_init` +parameters **must** be keyword-only. +Any arguments passed to the converter in Argument Clinic +will be passed along to your :py:meth:`!converter_init` method. +See :py:class:`!CConverter` for a list of members you may wish to specify in +your subclass. .. module:: clinic .. class:: CConverter + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + .. attribute:: type The C type to use for this variable. @@ -1436,16 +1438,16 @@ Here's the simplest example of a custom converter, from :source:`Modules/zlibmod [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. +This block adds a converter named ``ssize_t`` to Argument Clinic. +Parameters declared as ``ssize_t`` will be declared with type :c:type:`Py_ssize_t`, +and will be parsed by the ``'O&'`` format unit, +which will call the :c:func:`!ssize_t_converter` converter C function. +``ssize_t`` variables automatically support default values. More sophisticated custom converters can insert custom C code to handle initialization and cleanup. You can see more examples of custom converters in the CPython -source tree; grep the C files for the string :py:class:`!CConverter`. +source tree; grep the C files for the string ``CConverter``. How to write a custom return converter From 6996b406bcf9f6d85a59e539c743ef9126c3cc5d Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 5 Aug 2023 21:58:38 +0100 Subject: [PATCH 040/598] gh-104683: Improve consistency and test coverage of argument-clinic `__repr__` functions (#107667) --- Lib/test/test_clinic.py | 94 +++++++++++++++++++++++++++++++++++++++-- Tools/clinic/clinic.py | 20 +++++---- Tools/clinic/cpp.py | 7 ++- 3 files changed, 108 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 84b6a193ecf914..f30fad2126940b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -69,7 +69,7 @@ def __init__(self): self.converters = FakeConvertersDict() self.legacy_converters = FakeConvertersDict() self.language = clinic.CLanguage(None) - self.filename = None + self.filename = "clinic_tests" self.destination_buffers = {} self.block_parser = clinic.BlockParser('', self.language) self.modules = collections.OrderedDict() @@ -1849,10 +1849,10 @@ def test_non_ascii_character_in_docstring(self): self.parse(block) # The line numbers are off; this is a known limitation. expected = dedent("""\ - Warning on line 0: + Warning in file 'clinic_tests' on line 0: Non-ascii characters are not allowed in docstrings: 'á' - Warning on line 0: + Warning in file 'clinic_tests' on line 0: Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' """) @@ -3030,5 +3030,93 @@ def test_suffix_all_lines(self): self.assertEqual(out, expected) +class ClinicReprTests(unittest.TestCase): + def test_Block_repr(self): + block = clinic.Block("foo") + expected_repr = "" + self.assertEqual(repr(block), expected_repr) + + block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + expected_repr_2 = "" + self.assertEqual(repr(block2), expected_repr_2) + + block3 = clinic.Block( + input="longboi_" * 100, + dsl_name="wow_so_long", + signatures=[], + output="very_long_" * 100, + indent="" + ) + expected_repr_3 = ( + "" + ) + self.assertEqual(repr(block3), expected_repr_3) + + def test_Destination_repr(self): + destination = clinic.Destination( + "foo", type="file", clinic=FakeClinic(), args=("eggs",) + ) + self.assertEqual( + repr(destination), "" + ) + + destination2 = clinic.Destination("bar", type="buffer", clinic=FakeClinic()) + self.assertEqual(repr(destination2), "") + + def test_Module_repr(self): + module = clinic.Module("foo", FakeClinic()) + self.assertRegex(repr(module), r"") + + def test_Class_repr(self): + cls = clinic.Class("foo", FakeClinic(), None, 'some_typedef', 'some_type_object') + self.assertRegex(repr(cls), r"") + + def test_FunctionKind_repr(self): + self.assertEqual( + repr(clinic.FunctionKind.INVALID), "" + ) + self.assertEqual( + repr(clinic.FunctionKind.CLASS_METHOD), "" + ) + + def test_Function_and_Parameter_reprs(self): + function = clinic.Function( + name='foo', + module=FakeClinic(), + cls=None, + c_basename=None, + full_name='foofoo', + return_converter=clinic.init_return_converter(), + kind=clinic.FunctionKind.METHOD_INIT, + coexist=False + ) + self.assertEqual(repr(function), "") + + converter = clinic.self_converter('bar', 'bar', function) + parameter = clinic.Parameter( + "bar", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + function=function, + converter=converter + ) + self.assertEqual(repr(parameter), "") + + def test_Monitor_repr(self): + monitor = clinic.cpp.Monitor() + self.assertRegex(repr(monitor), r"") + + monitor.line_number = 42 + monitor.stack.append(("token1", "condition1")) + self.assertRegex( + repr(monitor), r"" + ) + + monitor.stack.append(("token2", "condition2")) + self.assertRegex( + repr(monitor), + r"" + ) + + if __name__ == "__main__": unittest.main() diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 7fbae1e0d870ad..423cdfb939c161 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1728,8 +1728,12 @@ def summarize(s: object) -> str: if len(s) > 30: return s[:26] + "..." + s[0] return s - return "".join(( - "")) + parts = ( + repr(dsl_name), + f"input={summarize(self.input)}", + f"output={summarize(self.output)}" + ) + return f"" class BlockParser: @@ -2037,10 +2041,10 @@ def __post_init__(self, args: tuple[str, ...]) -> None: def __repr__(self) -> str: if self.type == 'file': - file_repr = " " + repr(self.filename) + type_repr = f"type='file' file={self.filename!r}" else: - file_repr = '' - return "".join(("")) + type_repr = f"type={self.type!r}" + return f"" def clear(self) -> None: if self.type != 'buffer': @@ -2500,7 +2504,7 @@ def new_or_init(self) -> bool: return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} def __repr__(self) -> str: - return f"" + return f"" INVALID: Final = FunctionKind.INVALID @@ -2577,7 +2581,7 @@ def methoddef_flags(self) -> str | None: return '|'.join(flags) def __repr__(self) -> str: - return '' + return f'' def copy(self, **overrides: Any) -> Function: f = dc.replace(self, **overrides) @@ -2605,7 +2609,7 @@ class Parameter: right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: - return '' + return f'' def is_keyword_only(self) -> bool: return self.kind == inspect.Parameter.KEYWORD_ONLY diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 21a1b02e862c1c..876105120c97f2 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -43,9 +43,12 @@ def __post_init__(self) -> None: self.line_number = 0 def __repr__(self) -> str: - return ( - f"" + parts = ( + str(id(self)), + f"line={self.line_number}", + f"condition={self.condition()!r}" ) + return f"" def status(self) -> str: return str(self.line_number).rjust(4) + ": " + self.condition() From ecb05e0b9842ba03b42b4dec8767b1c18a4e28b3 Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Sun, 6 Aug 2023 14:10:50 +0900 Subject: [PATCH 041/598] GH-84435: Make pyspecific directives translatable (#19470) Co-authored-by: Jelle Zijlstra Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/tools/extensions/pyspecific.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8d99b0bfa4f381..765e6383ac6f5e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -98,14 +98,13 @@ class ImplementationDetail(Directive): final_argument_whitespace = True # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' + label_text = sphinx_gettext('CPython implementation detail:') def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = sphinx_gettext(self.label_text) content = self.content - add_text = nodes.strong(label, label) + add_text = nodes.strong(self.label_text, self.label_text) self.state.nested_parse(content, self.content_offset, pnode) content = nodes.inline(pnode[0].rawsource, translatable=True) content.source = pnode[0].source @@ -234,9 +233,9 @@ class AuditEvent(Directive): final_argument_whitespace = True _label = [ - "Raises an :ref:`auditing event ` {name} with no arguments.", - "Raises an :ref:`auditing event ` {name} with argument {args}.", - "Raises an :ref:`auditing event ` {name} with arguments {args}.", + sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), ] @property @@ -252,7 +251,7 @@ def run(self): else: args = [] - label = sphinx_gettext(self._label[min(2, len(args))]) + label = self._label[min(2, len(args))] text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -414,8 +413,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' + _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') + _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') def run(self): node = addnodes.versionmodified() @@ -431,7 +430,6 @@ def run(self): else: label = self._removed_label - label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], From 71a7c96ffeb0d7fef06be3e57468896e030967a5 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 6 Aug 2023 10:23:50 +0200 Subject: [PATCH 042/598] Docs: Fix Sphinx annotations in Doc/library/ctypes.rst (#107672) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/conf.py | 2 ++ Doc/library/ctypes.rst | 65 +++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 1b30b862f9a86c..49108eac58fc1d 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -110,6 +110,8 @@ ('c:type', 'uintptr_t'), ('c:type', 'va_list'), ('c:type', 'wchar_t'), + ('c:type', '__int64'), + ('c:type', 'unsigned __int64'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ea7873a885172e..ec4b0909181d32 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -72,8 +72,9 @@ Windows appends the usual ``.dll`` file suffix automatically. On Linux, it is required to specify the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: +:meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, +or you should load the library by creating an instance of CDLL by calling +the constructor:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -333,7 +334,7 @@ property:: 10 b'Hi\x00lo\x00\x00\x00\x00\x00' >>> -The :func:`create_string_buffer` function replaces the old :func:`c_buffer` +The :func:`create_string_buffer` function replaces the old :func:`!c_buffer` function (which is still available as an alias). To create a mutable memory block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -383,15 +384,15 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the *argtypes* attribute for the -regular, non-variadic, function arguments: +On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +attribute for the regular, non-variadic, function arguments: .. code-block:: python3 libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify ``argtypes`` for all variadic functions. +specify :attr:`~_FuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -401,7 +402,7 @@ Calling functions with your own custom data types You can also customize :mod:`ctypes` argument conversion to allow instances of your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. Of +:attr:`!_as_parameter_` attribute and uses this as the function argument. Of course, it must be one of integer, string, or bytes:: >>> class Bottles: @@ -414,7 +415,7 @@ course, it must be one of integer, string, or bytes:: 19 >>> -If you don't want to store the instance's data in the :attr:`_as_parameter_` +If you don't want to store the instance's data in the :attr:`!_as_parameter_` instance variable, you could define a :class:`property` which makes the attribute available on request. @@ -425,9 +426,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. +DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is +:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -451,14 +452,14 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives +to implement a :meth:`~_CData.from_param` class method for them to be able to use them +in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to +object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. +:attr:`!_as_parameter_` attribute. .. _ctypes-return-types: @@ -478,13 +479,13 @@ By default functions are assumed to return the C :c:expr:`int` type. Other return types can be specified by setting the :attr:`restype` attribute of the function object. -The C prototype of ``time()`` is ``time_t time(time_t *)``. Because :c:type:`time_t` -might be of a different type than the default return type ``int``, you should -specify the ``restype``:: +The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` +might be of a different type than the default return type :c:expr:`int`, you should +specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using ``argtypes``:: +The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -493,7 +494,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the ``strchr`` function, which expects +Here is a more advanced example, it uses the :func:`strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -506,8 +507,8 @@ a string pointer and a char, and returns a pointer to a string:: None >>> -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a +If you want to avoid the :func:`ord("x") ` calls above, you can set the +:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -853,7 +854,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -874,7 +875,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -1437,7 +1438,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` +parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1522,8 +1523,8 @@ underscore to not clash with exported function names: Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. +:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as +attribute of the loader instance. .. class:: LibraryLoader(dlltype) @@ -1639,14 +1640,14 @@ They are instances of a private class: unspecified arguments as well. When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` + :meth:`~_CData.from_param` class method of the items in the :attr:`argtypes` tuple, this method allows adapting the actual argument to an object that the foreign function accepts. For example, a :class:`c_char_p` item in the :attr:`argtypes` tuple will convert a string passed as argument into a bytes object using ctypes conversion rules. New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a + types, but each item must have a :meth:`~_CData.from_param` method which returns a value usable as argument (integer, string, ctypes instance). This allows defining adapters that can adapt custom objects as function parameters. @@ -1770,12 +1771,12 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`argtypes` tuple. + are specified in the :attr:`~_FuncPtr.argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. - *paramflags* must be a tuple of the same length as :attr:`argtypes`. + *paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. Each item in this tuple contains further information about a parameter, it must be a tuple containing one, two, or three items. @@ -2157,8 +2158,8 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. + foreign function's :attr:`~_FuncPtr.argtypes` tuple; + it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod that normally returns *obj* if that is an instance of the type. Some From 9641c4d8e2bdf9b00dd9f373d4a74dfad000afd1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 6 Aug 2023 13:08:32 +0200 Subject: [PATCH 043/598] Docs: skip python-docs-theme 2023.7 to fix mobile menu (#107666) --- Doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements.txt b/Doc/requirements.txt index e9b6e56b122895..d4f23ea8c400fe 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -15,6 +15,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2023.7 +python-docs-theme>=2023.3.1,!=2023.7 -c constraints.txt From 3e334ae259bd922733fac14f2ebac6e3dc93a5e1 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sun, 6 Aug 2023 14:37:12 +0200 Subject: [PATCH 044/598] gh-85160: improve performance of `functools.singledispatchmethod` (#107148) Co-authored-by: mental Co-authored-by: Alex Waygood --- Lib/functools.py | 21 +++++- Lib/test/test_functools.py | 68 +++++++++++++++++++ .../2020-11-10-07-04-15.bpo-40988.5kBC-O.rst | 3 + 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst diff --git a/Lib/functools.py b/Lib/functools.py index 8518450a8d499d..2a8a69b3c527aa 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -928,11 +928,14 @@ class singledispatchmethod: """ def __init__(self, func): + import weakref # see comment in singledispatch function if not callable(func) and not hasattr(func, "__get__"): raise TypeError(f"{func!r} is not callable or a descriptor") self.dispatcher = singledispatch(func) self.func = func + self._method_cache = weakref.WeakKeyDictionary() + self._all_weakrefable_instances = True def register(self, cls, method=None): """generic_method.register(cls, func) -> func @@ -942,13 +945,27 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): + if self._all_weakrefable_instances: + try: + _method = self._method_cache[obj] + except TypeError: + self._all_weakrefable_instances = False + except KeyError: + pass + else: + return _method + + dispatch = self.dispatcher.dispatch def _method(*args, **kwargs): - method = self.dispatcher.dispatch(args[0].__class__) - return method.__get__(obj, cls)(*args, **kwargs) + return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.register = self.register update_wrapper(_method, self.func) + + if self._all_weakrefable_instances: + self._method_cache[obj] = _method + return _method @property diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index c4eca0f5b79511..50770f066a5e16 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2474,6 +2474,74 @@ def _(arg): self.assertTrue(A.t('')) self.assertEqual(A.t(0.0), 0.0) + def test_slotted_class(self): + class Slot: + __slots__ = ('a', 'b') + @functools.singledispatchmethod + def go(self, item, arg): + pass + + @go.register + def _(self, item: int, arg): + return item + arg + + s = Slot() + self.assertEqual(s.go(1, 1), 2) + + def test_classmethod_slotted_class(self): + class Slot: + __slots__ = ('a', 'b') + @functools.singledispatchmethod + @classmethod + def go(cls, item, arg): + pass + + @go.register + @classmethod + def _(cls, item: int, arg): + return item + arg + + s = Slot() + self.assertEqual(s.go(1, 1), 2) + self.assertEqual(Slot.go(1, 1), 2) + + def test_staticmethod_slotted_class(self): + class A: + __slots__ = ['a'] + @functools.singledispatchmethod + @staticmethod + def t(arg): + return arg + @t.register(int) + @staticmethod + def _(arg): + return isinstance(arg, int) + @t.register(str) + @staticmethod + def _(arg): + return isinstance(arg, str) + a = A() + + self.assertTrue(A.t(0)) + self.assertTrue(A.t('')) + self.assertEqual(A.t(0.0), 0.0) + self.assertTrue(a.t(0)) + self.assertTrue(a.t('')) + self.assertEqual(a.t(0.0), 0.0) + + def test_assignment_behavior(self): + # see gh-106448 + class A: + @functools.singledispatchmethod + def t(arg): + return arg + + a = A() + a.t.foo = 'bar' + a2 = A() + with self.assertRaises(AttributeError): + a2.t.foo + def test_classmethod_register(self): class A: def __init__(self, arg): diff --git a/Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst b/Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst new file mode 100644 index 00000000000000..9323d93c59b05a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst @@ -0,0 +1,3 @@ +Improve performance of :class:`functools.singledispatchmethod` by caching the +generated dispatch wrapper. Optimization suggested by frederico. Patch by +@mental32, Alex Waygood and Pieter Eendebak. From 9564e31cbc95a723f2414537231bc4611b56644f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 6 Aug 2023 14:58:00 +0100 Subject: [PATCH 045/598] Do not use deprecated ``logger.warn()`` in pyspecific (#107694) --- Doc/tools/extensions/pyspecific.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 765e6383ac6f5e..3cf4d236604bcb 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -179,7 +179,7 @@ def parse_platforms(self): if unknown: cls = type(self) logger = logging.getLogger(cls.__qualname__) - logger.warn( + logger.warning( f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " f"in '.. availability:: {self.arguments[0]}', see " f"{__file__}:{cls.__qualname__}.known_platforms for a set " @@ -266,7 +266,7 @@ def run(self): info = env.all_audit_events.setdefault(name, new_info) if info is not new_info: if not self._do_args_match(info['args'], new_info['args']): - self.logger.warn( + self.logger.warning( "Mismatched arguments for audit-event {}: {!r} != {!r}" .format(name, info['args'], new_info['args']) ) @@ -542,7 +542,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.env.logger.warn('label %r not in documentation' % label) + self.env.logger.warning(f'label {label!r} not in documentation') continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) From ee3bf45e5e253d393f8553e58715c31e470800cd Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 6 Aug 2023 20:40:55 +0100 Subject: [PATCH 046/598] gh-106368: Improve coverage reports for argument clinic (#107693) --- .coveragerc | 5 +++++ Tools/clinic/clinic.py | 11 +++++++---- Tools/clinic/cpp.py | 10 ++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 18bf2f40fe523f..b5d94317e8aa8b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,11 @@ exclude_lines = # Don't complain if non-runnable code isn't run: if 0: if __name__ == .__main__.: + raise AssertionError\( + + # Empty bodies in protocols or abstract methods + ^\s*def [a-zA-Z0-9_]+\(.*\)(\s*->.*)?:\s*\.\.\.(\s*#.*)?$ + ^\s*\.\.\.(\s*#.*)?$ .*# pragma: no cover .*# pragma: no branch diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 423cdfb939c161..47b5f5ae32f581 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -469,7 +469,7 @@ class Language(metaclass=abc.ABCMeta): checksum_line = "" def __init__(self, filename: str) -> None: - pass + ... @abc.abstractmethod def render( @@ -477,10 +477,10 @@ def render( clinic: Clinic | None, signatures: Iterable[Module | Class | Function] ) -> str: - pass + ... def parse_line(self, line: str) -> None: - pass + ... def validate(self) -> None: def assert_only_one( @@ -2862,6 +2862,9 @@ def __getattr__(self, attr): f"Note: accessing self.function inside converter_init is disallowed!" ) return super().__getattr__(attr) + # this branch is just here for coverage reporting + else: # pragma: no cover + pass def converter_init(self) -> None: pass @@ -3990,7 +3993,7 @@ def correct_name_for_self( return "void *", "null" if f.kind in (CLASS_METHOD, METHOD_NEW): return "PyTypeObject *", "type" - raise RuntimeError("Unhandled type of function f: " + repr(f.kind)) + raise AssertionError(f"Unhandled type of function f: {f.kind!r}") def required_type_for_self_for_parser( f: Function diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 876105120c97f2..3d9c61ac678965 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -178,11 +178,17 @@ def pop_stack() -> TokenAndCondition: if self.verbose: print(self.status()) -if __name__ == '__main__': - for filename in sys.argv[1:]: + +def _main(filenames: list[str] | None = None) -> None: + filenames = filenames or sys.argv[1:] + for filename in filenames: with open(filename) as f: cpp = Monitor(filename, verbose=True) print() print(filename) for line_number, line in enumerate(f.read().split('\n'), 1): cpp.writeline(line) + + +if __name__ == '__main__': + _main() From 4e242d1ffb2d165443fe2680f7d1fef9fecbcfc0 Mon Sep 17 00:00:00 2001 From: Kamil Turek Date: Sun, 6 Aug 2023 23:11:16 +0200 Subject: [PATCH 047/598] Improve cross-references in `runpy` docs (#107673) - Add links to `__main__` and `sys.path` where appropriate - Ensure each paragraph never has more than one link to the same thing, to avoid visual clutter from too many links --- Doc/library/runpy.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 42ed8c253b8027..406b080b7be30f 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -39,7 +39,7 @@ The :mod:`runpy` module provides two functions: The *mod_name* argument should be an absolute module name. If the module name refers to a package rather than a normal - module, then that package is imported and the ``__main__`` submodule within + module, then that package is imported and the :mod:`__main__` submodule within that package is then executed and the resulting module globals dictionary returned. @@ -74,7 +74,7 @@ The :mod:`runpy` module provides two functions: Note that this manipulation of :mod:`sys` is not thread-safe. Other threads may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the :mod:`sys` module be left alone when + arguments. It is recommended that the ``sys`` module be left alone when invoking this function from threaded code. .. seealso:: @@ -82,7 +82,7 @@ The :mod:`runpy` module provides two functions: command line. .. versionchanged:: 3.1 - Added ability to execute packages by looking for a ``__main__`` submodule. + Added ability to execute packages by looking for a :mod:`__main__` submodule. .. versionchanged:: 3.2 Added ``__cached__`` global variable (see :pep:`3147`). @@ -106,15 +106,16 @@ The :mod:`runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid sys.path entry containing a ``__main__`` - module (e.g. a zipfile containing a top-level ``__main__.py`` file). + compiled bytecode file or a valid :data:`sys.path` entry containing a + :mod:`__main__` module + (e.g. a zipfile containing a top-level ``__main__.py`` file). For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid sys.path entry (typically a zipfile or + module namespace. For a valid :data:`sys.path` entry (typically a zipfile or directory), the entry is first added to the beginning of ``sys.path``. The function then looks for and executes a :mod:`__main__` module using the updated path. Note that there is no special protection against invoking - an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if + an existing ``__main__`` entry located elsewhere on ``sys.path`` if there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate @@ -137,14 +138,14 @@ The :mod:`runpy` module provides two functions: supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` + If the supplied path is a reference to a valid :data:`sys.path` entry, then + ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, - ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated + :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated with the value of ``path_name`` and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function @@ -152,7 +153,7 @@ The :mod:`runpy` module provides two functions: Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` are not optional in this function as these adjustments are essential to - allowing the execution of sys.path entries. As the thread-safety + allowing the execution of :data:`sys.path` entries. As the thread-safety limitations still apply, use of this function in threaded code should be either serialised with the import lock or delegated to a separate process. @@ -165,7 +166,7 @@ The :mod:`runpy` module provides two functions: .. versionchanged:: 3.4 Updated to take advantage of the module spec feature added by :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather + case where ``__main__`` is imported from a valid :data:`sys.path` entry rather than being executed directly. .. versionchanged:: 3.12 From a6675b1a597c67be972598ac8562883fabe48099 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 7 Aug 2023 02:08:34 +0200 Subject: [PATCH 048/598] Docs: Argument Clinic: Move the CConverter class to the reference (#107671) --- Doc/howto/clinic.rst | 133 ++++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 64 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index dcede13a03b4a5..e8e6aace350e0c 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -193,6 +193,71 @@ The CLI supports the following options: The list of files to process. +.. _clinic-classes: + +Classes for extending Argument Clinic +------------------------------------- + +.. module:: clinic + +.. class:: CConverter + + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + + .. attribute:: type + + The C type to use for this variable. + :attr:`!type` should be a Python string specifying the type, + e.g. ``'int'``. + If this is a pointer type, the type string should end with ``' *'``. + + .. attribute:: default + + The Python default value for this parameter, as a Python value. + Or the magic value ``unspecified`` if there is no default. + + .. attribute:: py_default + + :attr:`!default` as it should appear in Python code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_default + + :attr:`!default` as it should appear in C code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_ignored_default + + The default value used to initialize the C variable when + there is no default, but not specifying a default may + result in an "uninitialized variable" warning. This can + easily happen when using option groups—although + properly written code will never actually use this value, + the variable does get passed in to the impl, and the + C compiler will complain about the "use" of the + uninitialized value. This value should always be a + non-empty string. + + .. attribute:: converter + + The name of the C converter function, as a string. + + .. attribute:: impl_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into the impl function. + + .. attribute:: parse_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into :c:func:`PyArg_ParseTuple`. + + .. _clinic-tutorial: Tutorial @@ -1348,7 +1413,7 @@ See also :pep:`573`. How to write a custom converter ------------------------------- -A converter is a Python class that inherits from :py:class:`!CConverter`. +A converter is a Python class that inherits from :py:class:`CConverter`. The main purpose of a custom converter, is for parameters parsed with the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". @@ -1360,73 +1425,13 @@ your converter class with the ``_converter`` suffix stripped off. Instead of subclassing :py:meth:`!CConverter.__init__`, write a :py:meth:`!converter_init` method. -Apart for the *self* parameter, all additional :py:meth:`!converter_init` -parameters **must** be keyword-only. +:py:meth:`!converter_init` always accepts a *self* parameter. +After *self*, all additional parameters **must** be keyword-only. Any arguments passed to the converter in Argument Clinic will be passed along to your :py:meth:`!converter_init` method. -See :py:class:`!CConverter` for a list of members you may wish to specify in +See :py:class:`CConverter` for a list of members you may wish to specify in your subclass. -.. module:: clinic - -.. class:: CConverter - - The base class for all converters. - See :ref:`clinic-howto-custom-converter` for how to subclass this class. - - .. attribute:: type - - The C type to use for this variable. - :attr:`!type` should be a Python string specifying the type, - e.g. ``'int'``. - If this is a pointer type, the type string should end with ``' *'``. - - .. attribute:: default - - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - - .. attribute:: py_default - - :attr:`!default` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_default - - :attr:`!default` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_ignored_default - - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups—although - properly written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - - .. attribute:: converter - - The name of the C converter function, as a string. - - .. attribute:: impl_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - - .. attribute:: parse_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - Here's the simplest example of a custom converter, from :source:`Modules/zlibmodule.c`:: /*[python input] From 6925c578a0e3cbb00858e64da813a7ffe79623c4 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Mon, 7 Aug 2023 12:41:39 +0200 Subject: [PATCH 049/598] gh-107442: Document all valid types for ctypes _as_parameter_ (#107443) --- Doc/library/ctypes.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ec4b0909181d32..fcf711efe0eb7d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -401,9 +401,10 @@ Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`!_as_parameter_` attribute and uses this as the function argument. Of -course, it must be one of integer, string, or bytes:: +your own classes be used as function arguments. :mod:`ctypes` looks for an +:attr:`!_as_parameter_` attribute and uses this as the function argument. The +attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: ... def __init__(self, number): From 3c8e8f3ceeae08fc43d885f5a4c65a3ee4b1a2c8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 14:11:39 +0300 Subject: [PATCH 050/598] gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) In future Tcl and Tk versions can be desynchronized. --- Lib/test/test_tcl.py | 13 ++----------- Lib/test/test_tkinter/support.py | 18 +++++++++--------- Lib/test/test_tkinter/test_images.py | 6 +++--- Lib/test/test_tkinter/test_widgets.py | 16 ++++++++-------- Lib/test/test_tkinter/widget_tests.py | 6 +++--- Lib/test/test_ttk/test_style.py | 2 +- Lib/test/test_ttk/test_widgets.py | 8 ++++---- 7 files changed, 30 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index d07b83acb1b505..ebdb58f91d3d8a 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -20,14 +20,6 @@ tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) -_tk_patchlevel = None -def get_tk_patchlevel(): - global _tk_patchlevel - if _tk_patchlevel is None: - tcl = Tcl() - _tk_patchlevel = tcl.info_patchlevel() - return _tk_patchlevel - class TkinterTest(unittest.TestCase): @@ -571,7 +563,6 @@ def test_splitlist(self): (1, '2', (3.4,)) if self.wantobjects else ('1', '2', '3.4')), ] - tk_patchlevel = get_tk_patchlevel() if not self.wantobjects: expected = ('12', '\u20ac', '\xe2\x82\xac', '3.4') else: @@ -580,8 +571,8 @@ def test_splitlist(self): (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), expected), ] - dbg_info = ('want objects? %s, Tcl version: %s, Tk patchlevel: %s' - % (self.wantobjects, tcl_version, tk_patchlevel)) + dbg_info = ('want objects? %s, Tcl version: %s, Tcl patchlevel: %s' + % (self.wantobjects, tcl_version, self.interp.info_patchlevel())) for arg, res in testcases: self.assertEqual(splitlist(arg), res, 'arg=%a, %s' % (arg, dbg_info)) diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index 9154ebac5c48f8..10e64bf40a4afa 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -79,28 +79,28 @@ def simulate_mouse_click(widget, x, y): import _tkinter tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) +tk_version = tuple(map(int, _tkinter.TK_VERSION.split('.'))) -def requires_tcl(*version): - if len(version) <= 2: - return unittest.skipUnless(tcl_version >= version, - 'requires Tcl version >= ' + '.'.join(map(str, version))) +def requires_tk(*version): + if len(version) <= 2 and tk_version >= version: + return lambda test: test def deco(test): @functools.wraps(test) def newtest(self): - if get_tk_patchlevel() < version: - self.skipTest('requires Tcl version >= ' + + root = getattr(self, 'root', None) + if get_tk_patchlevel(root) < version: + self.skipTest('requires Tk version >= ' + '.'.join(map(str, version))) test(self) return newtest return deco _tk_patchlevel = None -def get_tk_patchlevel(): +def get_tk_patchlevel(root): global _tk_patchlevel if _tk_patchlevel is None: - tcl = tkinter.Tcl() - _tk_patchlevel = tcl.info_patchlevel() + _tk_patchlevel = tkinter._parse_version(root.tk.globalgetvar('tk_patchLevel')) return _tk_patchlevel units = { diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index c07de867ce04b7..b6ba1c22dcbaca 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -2,7 +2,7 @@ import tkinter from test import support from test.support import os_helper -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tcl +from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -213,11 +213,11 @@ def test_create_from_gif_file(self): def test_create_from_gif_data(self): self.check_create_from_data('gif') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_file(self): self.check_create_from_file('png') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_data(self): self.check_create_from_data('png') diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 34e67c0cbc44a3..d3f942db7baf9a 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,7 +4,7 @@ import os from test.support import requires -from test.test_tkinter.support import (requires_tcl, +from test.test_tkinter.support import (requires_tk, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( @@ -613,7 +613,7 @@ def test_configure_inactiveselectbackground(self): widget = self.create() self.checkColorParam(widget, 'inactiveselectbackground') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_configure_insertunfocussed(self): widget = self.create() self.checkEnumParam(widget, 'insertunfocussed', @@ -924,7 +924,7 @@ def test_coords(self): for i in range(4): self.assertIsInstance(coords[i], float) - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_moveto(self): widget = self.create() i1 = widget.create_rectangle(1, 1, 20, 20, tags='group') @@ -969,7 +969,7 @@ def test_configure_activestyle(self): self.checkEnumParam(widget, 'activestyle', 'dotbox', 'none', 'underline') - test_configure_justify = requires_tcl(8, 6, 5)(StandardOptionsTests.test_configure_justify) + test_configure_justify = requires_tk(8, 6, 5)(StandardOptionsTests.test_configure_justify) def test_configure_listvariable(self): widget = self.create() @@ -1108,7 +1108,7 @@ def test_configure_digits(self): def test_configure_from(self): widget = self.create() - conv = float if get_tk_patchlevel() >= (8, 6, 10) else float_round + conv = float if get_tk_patchlevel(self.root) >= (8, 6, 10) else float_round self.checkFloatParam(widget, 'from', 100, 14.9, 15.1, conv=conv) def test_configure_label(self): @@ -1235,19 +1235,19 @@ def test_configure_opaqueresize(self): widget = self.create() self.checkBooleanParam(widget, 'opaqueresize') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxybackground(self): widget = self.create() self.checkColorParam(widget, 'proxybackground') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'proxyborderwidth', 0, 1.3, 2.9, 6, -2, '10p', conv=False) - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() self.checkReliefParam(widget, 'proxyrelief') diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index f60087a6e9f385..31f82f459beefd 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -1,7 +1,7 @@ # Common tests for test_tkinter/test_widgets.py and test_ttk/test_widgets.py import tkinter -from test.test_tkinter.support import (AbstractTkTest, tcl_version, +from test.test_tkinter.support import (AbstractTkTest, tk_version, pixels_conv, tcl_obj_eq) import test.support @@ -22,7 +22,7 @@ def scaling(self): return self._scaling def _str(self, value): - if not self._stringify and self.wantobjects and tcl_version >= (8, 6): + if not self._stringify and self.wantobjects and tk_version >= (8, 6): return value if isinstance(value, tuple): return ' '.join(map(self._str, value)) @@ -156,7 +156,7 @@ def checkReliefParam(self, widget, name): 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') errmsg='bad relief "spam": must be '\ 'flat, groove, raised, ridge, solid, or sunken' - if tcl_version < (8, 6): + if tk_version < (8, 6): errmsg = None self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 0ec95cf6b5ffc9..f9c56ec2357451 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -170,7 +170,7 @@ def test_map_custom_copy(self): newname = f'C.{name}' self.assertEqual(style.map(newname), {}) style.map(newname, **default) - if theme == 'alt' and name == '.' and get_tk_patchlevel() < (8, 6, 1): + if theme == 'alt' and name == '.' and get_tk_patchlevel(self.root) < (8, 6, 1): default['embossed'] = [('disabled', '1')] self.assertEqual(style.map(newname), default) for key, value in default.items(): diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 79d65b496abdc6..fd1a748a498ac5 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,7 +5,7 @@ import sys from test.test_ttk_textonly import MockTclObj -from test.test_tkinter.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, +from test.test_tkinter.support import (AbstractTkTest, tk_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) @@ -19,7 +19,7 @@ def test_configure_class(self): widget = self.create() self.assertEqual(widget['class'], '') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) widget2 = self.create(class_='Foo') @@ -560,7 +560,7 @@ def test_configure_orient(self): widget = self.create() self.assertEqual(str(widget['orient']), 'vertical') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'orient', 'horizontal', errmsg=errmsg) @@ -1526,7 +1526,7 @@ def test_heading(self): def test_heading_callback(self): def simulate_heading_click(x, y): - if tcl_version >= (8, 6): + if tk_version >= (8, 6): self.assertEqual(self.tv.identify_column(x), '#0') self.assertEqual(self.tv.identify_region(x, y), 'heading') simulate_mouse_click(self.tv, x, y) From 33cb0b06efe33968eb32463fa1b02b5a729a17f8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 7 Aug 2023 13:28:08 +0200 Subject: [PATCH 051/598] gh-95065: Add Argument Clinic support for deprecating positional use of parameters (#95151) It is now possible to deprecate passing parameters positionally with Argument Clinic, using the new '* [from X.Y]' syntax. (To be read as "keyword-only from Python version X.Y") Co-authored-by: Alex Waygood Co-authored-by: Serhiy Storchaka --- Doc/howto/clinic.rst | 88 ++ Lib/test/clinic.test.c | 810 ++++++++++++++++++ Lib/test/test_clinic.py | 96 ++- ...2-07-23-00-33-28.gh-issue-95065.NfCCpp.rst | 6 + Tools/clinic/clinic.py | 167 +++- 5 files changed, 1153 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index e8e6aace350e0c..286623c2410145 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1898,3 +1898,91 @@ blocks embedded in Python files look slightly different. They look like this: #[python start generated code]*/ def foo(): pass #/*[python checksum:...]*/ + + +.. _clinic-howto-deprecate-positional: + +How to deprecate passing parameters positionally +------------------------------------------------ + +Argument Clinic provides syntax that makes it possible to generate code that +deprecates passing :term:`arguments ` positionally. +For example, say we've got a module-level function :py:func:`!foo.myfunc` +that has three :term:`parameters `: +positional-or-keyword parameters *a* and *b*, and a keyword-only parameter *c*:: + + /*[clinic input] + module foo + myfunc + a: int + b: int + * + c: int + [clinic start generated output]*/ + +We now want to make the *b* parameter keyword-only; +however, we'll have to wait two releases before making this change, +as mandated by Python's backwards-compatibility policy (see :pep:`387`). +For this example, imagine we're in the development phase for Python 3.12: +that means we'll be allowed to introduce deprecation warnings in Python 3.12 +whenever the *b* parameter is passed positionally, +and we'll be allowed to make it keyword-only in Python 3.14 at the earliest. + +We can use Argument Clinic to emit the desired deprecation warnings +using the ``* [from ...]``` syntax, +by adding the line ``* [from 3.14]`` right above the *b* parameter:: + + /*[clinic input] + module foo + myfunc + a: int + * [from 3.14] + b: int + * + c: int + [clinic start generated output]*/ + +Next, regenerate Argument Clinic code (``make clinic``), +and add unit tests for the new behaviour. + +The generated code will now emit a :exc:`DeprecationWarning` +when an :term:`argument` for the :term:`parameter` *b* is passed positionally. +C preprocessor directives are also generated for emitting +compiler warnings if the ``* [from ...]`` line has not been removed +from the Argument Clinic input when the deprecation period is over, +which means when the alpha phase of the specified Python version kicks in. + +Let's return to our example and skip ahead two years: +Python 3.14 development has now entered the alpha phase, +but we forgot all about updating the Argument Clinic code +for :py:func:`!myfunc`! +Luckily for us, compiler warnings are now generated: + +.. code-block:: none + + In file included from Modules/foomodule.c:139: + Modules/clinic/foomodule.c.h:83:8: warning: Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only. [-W#warnings] + # warning "Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only." + ^ + +We now close the deprecation phase by making *b* keyword-only; +replace the ``* [from ...]``` line above *b* +with the ``*`` from the line above *c*:: + + /*[clinic input] + module foo + myfunc + a: int + * + b: int + c: int + [clinic start generated output]*/ + +Finally, run ``make clinic`` to regenerate the Argument Clinic code, +and update your unit tests to reflect the new behaviour. + +.. note:: + + If you forget to update your input block during the alpha and beta phases, + the compiler warning will turn into a compiler error when the + release candidate phase begins. diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index d2ad1a0482c304..321ac69273189f 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5380,6 +5380,7 @@ static PyObject * fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) /*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ + /*[python input] class Custom_converter(CConverter): type = "str" @@ -5464,3 +5465,812 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) /*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ + + +/*[clinic input] +test_deprecate_positional_pos1_len1_optional + a: object + * [from 3.14] + b: object = None +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos1_len1_optional__doc__, +"test_deprecate_positional_pos1_len1_optional($module, /, a, b=None)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_OPTIONAL_METHODDEF \ + {"test_deprecate_positional_pos1_len1_optional", _PyCFunction_CAST(test_deprecate_positional_pos1_len1_optional), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1_optional__doc__}, + +static PyObject * +test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, + PyObject *a, PyObject *b); + +static PyObject * +test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos1_len1_optional", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1_optional() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = test_deprecate_positional_pos1_len1_optional_impl(module, a, b); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, + PyObject *a, PyObject *b) +/*[clinic end generated code: output=20bdea6a2960ddf3 input=89099f3dacd757da]*/ + + +/*[clinic input] +test_deprecate_positional_pos1_len1 + a: object + * [from 3.14] + b: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos1_len1__doc__, +"test_deprecate_positional_pos1_len1($module, /, a, b)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_METHODDEF \ + {"test_deprecate_positional_pos1_len1", _PyCFunction_CAST(test_deprecate_positional_pos1_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1__doc__}, + +static PyObject * +test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, + PyObject *b); + +static PyObject * +test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos1_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = test_deprecate_positional_pos1_len1_impl(module, a, b); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, + PyObject *b) +/*[clinic end generated code: output=22c70f8b36085758 input=1702bbab1e9b3b99]*/ + + +/*[clinic input] +test_deprecate_positional_pos1_len2_with_kwd + a: object + * [from 3.14] + b: object + c: object + * + d: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos1_len2_with_kwd__doc__, +"test_deprecate_positional_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS1_LEN2_WITH_KWD_METHODDEF \ + {"test_deprecate_positional_pos1_len2_with_kwd", _PyCFunction_CAST(test_deprecate_positional_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len2_with_kwd__doc__}, + +static PyObject * +test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, + PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos1_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + # endif + #endif + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to test_deprecate_positional_pos1_len2_with_kwd() is deprecated. Parameters 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = test_deprecate_positional_pos1_len2_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, + PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=79c5f04220a1f3aa input=28cdb885f6c34eab]*/ + + +/*[clinic input] +test_deprecate_positional_pos0_len1 + * [from 3.14] + a: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos0_len1__doc__, +"test_deprecate_positional_pos0_len1($module, /, a)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS0_LEN1_METHODDEF \ + {"test_deprecate_positional_pos0_len1", _PyCFunction_CAST(test_deprecate_positional_pos0_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len1__doc__}, + +static PyObject * +test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a); + +static PyObject * +test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos0_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len1() is deprecated. Parameter 'a' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = test_deprecate_positional_pos0_len1_impl(module, a); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=1b7f23b9ffca431b input=678206db25c0652c]*/ + + +/*[clinic input] +test_deprecate_positional_pos0_len2 + * [from 3.14] + a: object + b: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos0_len2__doc__, +"test_deprecate_positional_pos0_len2($module, /, a, b)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS0_LEN2_METHODDEF \ + {"test_deprecate_positional_pos0_len2", _PyCFunction_CAST(test_deprecate_positional_pos0_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len2__doc__}, + +static PyObject * +test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, + PyObject *b); + +static PyObject * +test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos0_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len2() is deprecated. Parameters 'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = test_deprecate_positional_pos0_len2_impl(module, a, b); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, + PyObject *b) +/*[clinic end generated code: output=31b494f2dcc016af input=fae0d0b1d480c939]*/ + + +/*[clinic input] +test_deprecate_positional_pos0_len3_with_kwdonly + * [from 3.14] + a: object + b: object + c: object + * + e: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos0_len3_with_kwdonly__doc__, +"test_deprecate_positional_pos0_len3_with_kwdonly($module, /, a, b, c,\n" +" *, e)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS0_LEN3_WITH_KWDONLY_METHODDEF \ + {"test_deprecate_positional_pos0_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos0_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len3_with_kwdonly__doc__}, + +static PyObject * +test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *e); + +static PyObject * +test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos0_len3_with_kwdonly", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *e; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated. Parameters 'a', 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + e = args[3]; + return_value = test_deprecate_positional_pos0_len3_with_kwdonly_impl(module, a, b, c, e); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *e) +/*[clinic end generated code: output=96978e786acfbc7b input=1b0121770c0c52e0]*/ + + +/*[clinic input] +test_deprecate_positional_pos2_len1 + a: object + b: object + * [from 3.14] + c: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos2_len1__doc__, +"test_deprecate_positional_pos2_len1($module, /, a, b, c)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS2_LEN1_METHODDEF \ + {"test_deprecate_positional_pos2_len1", _PyCFunction_CAST(test_deprecate_positional_pos2_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len1__doc__}, + +static PyObject * +test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c); + +static PyObject * +test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos2_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + # endif + #endif + if (nargs == 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 3 positional arguments to test_deprecate_positional_pos2_len1() is deprecated. Parameter 'c' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = test_deprecate_positional_pos2_len1_impl(module, a, b, c); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=ceadd05f11f7f491 input=e1d129689e69ec7c]*/ + + +/*[clinic input] +test_deprecate_positional_pos2_len2 + a: object + b: object + * [from 3.14] + c: object + d: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos2_len2__doc__, +"test_deprecate_positional_pos2_len2($module, /, a, b, c, d)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS2_LEN2_METHODDEF \ + {"test_deprecate_positional_pos2_len2", _PyCFunction_CAST(test_deprecate_positional_pos2_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len2__doc__}, + +static PyObject * +test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos2_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len2() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = test_deprecate_positional_pos2_len2_impl(module, a, b, c, d); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=5693682e3fa1188b input=0d53533463a12792]*/ + + +/*[clinic input] +test_deprecate_positional_pos2_len3_with_kwdonly + a: object + b: object + * [from 3.14] + c: object + d: object + * + e: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos2_len3_with_kwdonly__doc__, +"test_deprecate_positional_pos2_len3_with_kwdonly($module, /, a, b, c,\n" +" d, *, e)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS2_LEN3_WITH_KWDONLY_METHODDEF \ + {"test_deprecate_positional_pos2_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos2_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len3_with_kwdonly__doc__}, + +static PyObject * +test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *d, + PyObject *e); + +static PyObject * +test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos2_len3_with_kwdonly", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + return_value = test_deprecate_positional_pos2_len3_with_kwdonly_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *d, + PyObject *e) +/*[clinic end generated code: output=00d436de747a00f3 input=154fd450448d8935]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f30fad2126940b..f594e39a90546a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1478,11 +1478,105 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - err = "Function 'bar' specifies '*' without any parameters afterwards." + err = "Function 'foo.bar' specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): self.expect_failure(block, err) + def test_parameters_required_after_depr_star(self): + dataset = ( + "module foo\nfoo.bar\n * [from 3.14]", + "module foo\nfoo.bar\n * [from 3.14]\nDocstring here.", + "module foo\nfoo.bar\n this: int\n * [from 3.14]", + "module foo\nfoo.bar\n this: int\n * [from 3.14]\nDocstring.", + ) + err = "Function 'foo.bar' specifies '* [from 3.14]' without any parameters afterwards." + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_depr_star_invalid_format_1(self): + block = """ + module foo + foo.bar + this: int + * [from 3] + Docstring. + """ + err = ( + "Function 'foo.bar': expected format '* [from major.minor]' " + "where 'major' and 'minor' are integers; got '3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_2(self): + block = """ + module foo + foo.bar + this: int + * [from a.b] + Docstring. + """ + err = ( + "Function 'foo.bar': expected format '* [from major.minor]' " + "where 'major' and 'minor' are integers; got 'a.b'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_3(self): + block = """ + module foo + foo.bar + this: int + * [from 1.2.3] + Docstring. + """ + err = ( + "Function 'foo.bar': expected format '* [from major.minor]' " + "where 'major' and 'minor' are integers; got '1.2.3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_parameters_required_after_depr_star(self): + block = """ + module foo + foo.bar + this: int + * [from 3.14] + Docstring. + """ + err = ( + "Function 'foo.bar' specifies '* [from ...]' without " + "any parameters afterwards" + ) + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_star(self): + block = """ + module foo + foo.bar + this: int + * + * [from 3.14] + Docstring. + """ + err = "Function 'foo.bar': '* [from ...]' must come before '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_duplicate(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.14] + c: int + Docstring. + """ + err = "Function 'foo.bar' uses '[from ...]' more than once" + self.expect_failure(block, err, lineno=5) + def test_single_slash(self): block = """ module foo diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst b/Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst new file mode 100644 index 00000000000000..3641716769cd56 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst @@ -0,0 +1,6 @@ +It is now possible to deprecate passing parameters positionally with +Argument Clinic, using the new ``* [from X.Y]`` syntax. +(To be read as *"keyword-only from Python version X.Y"*.) +See :ref:`clinic-howto-deprecate-positional` for more information. +Patch by Erlend E. Aasland with help from Alex Waygood, +Nikita Sobolev, and Serhiy Storchaka. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 47b5f5ae32f581..4dfe90b314f543 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -347,6 +347,13 @@ def suffix_all_lines(s: str, suffix: str) -> str: return ''.join(final) +def pprint_words(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + else: + return ", ".join(items[:-1]) + " and " + items[-1] + + def version_splitter(s: str) -> tuple[int, ...]: """Splits a version string into a tuple of integers. @@ -828,6 +835,22 @@ class CLanguage(Language): #define {methoddef_name} #endif /* !defined({methoddef_name}) */ """) + DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" + #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 + # error "{cpp_message}" + #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 + # ifdef _MSC_VER + # pragma message ("{cpp_message}") + # else + # warning "{cpp_message}" + # endif + #endif + if ({condition}) {{{{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, "{depr_message}", 1)) {{{{ + goto exit; + }}}} + }}}} + """ def __init__(self, filename: str) -> None: super().__init__(filename) @@ -850,6 +873,64 @@ def render( function = o return self.render_function(clinic, function) + def deprecate_positional_use( + self, + func: Function, + params: dict[int, Parameter], + ) -> str: + assert len(params) > 0 + names = [repr(p.name) for p in params.values()] + first_pos, first_param = next(iter(params.items())) + last_pos, last_param = next(reversed(params.items())) + + # Pretty-print list of names. + pstr = pprint_words(names) + + # For now, assume there's only one deprecation level. + assert first_param.deprecated_positional == last_param.deprecated_positional + thenceforth = first_param.deprecated_positional + assert thenceforth is not None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + source = os.path.basename(self.cpp.filename) + major, minor = thenceforth + cpp_message = ( + f"In {source}, update parameter(s) {pstr} in the clinic " + f"input of {func.full_name!r} to be keyword-only." + ) + # Format the deprecation message. + if first_pos == 0: + preamble = "Passing positional arguments to " + if len(params) == 1: + condition = f"nargs == {first_pos+1}" + if first_pos: + preamble = f"Passing {first_pos+1} positional arguments to " + depr_message = preamble + ( + f"{func.full_name}() is deprecated. Parameter {pstr} will " + f"become a keyword-only parameter in Python {major}.{minor}." + ) + else: + condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" + if first_pos: + preamble = ( + f"Passing more than {first_pos} positional " + f"argument{'s' if first_pos != 1 else ''} to " + ) + depr_message = preamble + ( + f"{func.full_name}() is deprecated. Parameters {pstr} will " + f"become keyword-only parameters in Python {major}.{minor}." + ) + # Format and return the code block. + code = self.DEPRECATED_POSITIONAL_PROTOTYPE.format( + condition=condition, + major=major, + minor=minor, + cpp_message=cpp_message, + depr_message=depr_message, + ) + return normalize_snippet(code, indent=4) + def docstring_for_c_string( self, f: Function @@ -1199,6 +1280,7 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS + deprecated_positionals: dict[int, Parameter] = {} add_label: str | None = None for i, p in enumerate(parameters): if isinstance(p.converter, defining_class_converter): @@ -1213,6 +1295,8 @@ def parser_body( parser_code.append("%s:" % add_label) add_label = None if not p.is_optional(): + if p.deprecated_positional: + deprecated_positionals[i] = p parser_code.append(normalize_snippet(parsearg, indent=4)) elif i < pos_only: add_label = 'skip_optional_posonly' @@ -1242,6 +1326,8 @@ def parser_body( goto %s; }} """ % add_label, indent=4)) + if p.deprecated_positional: + deprecated_positionals[i] = p if i + 1 == len(parameters): parser_code.append(normalize_snippet(parsearg, indent=4)) else: @@ -1257,6 +1343,12 @@ def parser_body( }} """ % add_label, indent=4)) + if deprecated_positionals: + code = self.deprecate_positional_use(f, deprecated_positionals) + assert parser_code is not None + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + if parser_code is not None: if add_label: parser_code.append("%s:" % add_label) @@ -2592,6 +2684,9 @@ def copy(self, **overrides: Any) -> Function: return f +VersionTuple = tuple[int, int] + + @dc.dataclass(repr=False, slots=True) class Parameter: """ @@ -2606,6 +2701,8 @@ class Parameter: annotation: object = inspect.Parameter.empty docstring: str = '' group: int = 0 + # (`None` signifies that there is no deprecation) + deprecated_positional: VersionTuple | None = None right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: @@ -4430,6 +4527,7 @@ class DSLParser: state: StateKeeper keyword_only: bool positional_only: bool + deprecated_positional: VersionTuple | None group: int parameter_state: ParamState indent: IndentStack @@ -4437,6 +4535,11 @@ class DSLParser: coexist: bool parameter_continuation: str preserve_output: bool + star_from_version_re = create_regex( + before="* [from ", + after="]", + word=False, + ) def __init__(self, clinic: Clinic) -> None: self.clinic = clinic @@ -4460,6 +4563,7 @@ def reset(self) -> None: self.state = self.state_dsl_start self.keyword_only = False self.positional_only = False + self.deprecated_positional = None self.group = 0 self.parameter_state: ParamState = ParamState.START self.indent = IndentStack() @@ -4622,7 +4726,7 @@ def parse(self, block: Block) -> None: exc.lineno = line_number raise - self.do_post_block_processing_cleanup() + self.do_post_block_processing_cleanup(line_number) block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) if self.preserve_output: @@ -4908,8 +5012,14 @@ def state_parameter(self, line: str) -> None: self.parameter_continuation = line[:-1] return + line = line.lstrip() + match = self.star_from_version_re.match(line) + if match: + self.parse_deprecated_positional(match.group(1)) + return + func = self.function - match line.lstrip(): + match line: case '*': self.parse_star(func) case '[': @@ -5182,7 +5292,9 @@ def bad_node(self, node: ast.AST) -> None: "after 'self'.") - p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) + p = Parameter(parameter_name, kind, function=self.function, + converter=converter, default=value, group=self.group, + deprecated_positional=self.deprecated_positional) names = [k.name for k in self.function.parameters.values()] if parameter_name in names[1:]: @@ -5215,10 +5327,28 @@ def parse_converter( "Annotations must be either a name, a function call, or a string." ) + def parse_deprecated_positional(self, thenceforth: str) -> None: + assert isinstance(self.function, Function) + fname = self.function.full_name + + if self.keyword_only: + fail(f"Function {fname!r}: '* [from ...]' must come before '*'") + if self.deprecated_positional: + fail(f"Function {fname!r} uses '[from ...]' more than once.") + try: + major, minor = thenceforth.split(".") + self.deprecated_positional = int(major), int(minor) + except ValueError: + fail( + f"Function {fname!r}: expected format '* [from major.minor]' " + f"where 'major' and 'minor' are integers; got {thenceforth!r}" + ) + def parse_star(self, function: Function) -> None: """Parse keyword-only parameter marker '*'.""" if self.keyword_only: fail(f"Function {function.name!r} uses '*' more than once.") + self.deprecated_positional = None self.keyword_only = True def parse_opening_square_bracket(self, function: Function) -> None: @@ -5586,23 +5716,34 @@ def add_parameter(text: str) -> None: return docstring - def do_post_block_processing_cleanup(self) -> None: + def do_post_block_processing_cleanup(self, lineno: int) -> None: """ Called when processing the block is done. """ if not self.function: return - if self.keyword_only: - values = self.function.parameters.values() - if not values: - no_parameter_after_star = True + def check_remaining( + symbol: str, + condition: Callable[[Parameter], bool] + ) -> None: + assert isinstance(self.function, Function) + + if values := self.function.parameters.values(): + last_param = next(reversed(values)) + no_param_after_symbol = condition(last_param) else: - last_parameter = next(reversed(list(values))) - no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY - if no_parameter_after_star: - fail(f"Function {self.function.name!r} specifies '*' " - "without any parameters afterwards.") + no_param_after_symbol = True + if no_param_after_symbol: + fname = self.function.full_name + fail(f"Function {fname!r} specifies {symbol!r} " + "without any parameters afterwards.", line_number=lineno) + + if self.keyword_only: + check_remaining("*", lambda p: p.kind != inspect.Parameter.KEYWORD_ONLY) + + if self.deprecated_positional: + check_remaining("* [from ...]", lambda p: not p.deprecated_positional) self.function.docstring = self.format_docstring() From 2ac103c346ffe9d0e4c146402ce215c5ce6c1ef2 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 7 Aug 2023 13:46:36 +0100 Subject: [PATCH 052/598] gh-85160: Reduce memory usage of `singledispatchmethod` (#107706) A small followup to #107148 Co-authored-by: Serhiy Storchaka --- Lib/functools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 2a8a69b3c527aa..be44ccdae6b692 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -928,14 +928,14 @@ class singledispatchmethod: """ def __init__(self, func): - import weakref # see comment in singledispatch function if not callable(func) and not hasattr(func, "__get__"): raise TypeError(f"{func!r} is not callable or a descriptor") self.dispatcher = singledispatch(func) self.func = func + + import weakref # see comment in singledispatch function self._method_cache = weakref.WeakKeyDictionary() - self._all_weakrefable_instances = True def register(self, cls, method=None): """generic_method.register(cls, func) -> func @@ -945,11 +945,11 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): - if self._all_weakrefable_instances: + if self._method_cache is not None: try: _method = self._method_cache[obj] except TypeError: - self._all_weakrefable_instances = False + self._method_cache = None except KeyError: pass else: @@ -963,7 +963,7 @@ def _method(*args, **kwargs): _method.register = self.register update_wrapper(_method, self.func) - if self._all_weakrefable_instances: + if self._method_cache is not None: self._method_cache[obj] = _method return _method From 8c9af6b9a0d6fc9cb237e96588d8dcab727e32b8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 7 Aug 2023 15:11:05 +0200 Subject: [PATCH 053/598] Docs: Fix more Sphinx annotations in ctypes.rst (#107708) --- Doc/conf.py | 1 + Doc/library/ctypes.rst | 85 +++++++++++++++++++++--------------------- Doc/tools/.nitignore | 1 - 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 49108eac58fc1d..19e05e1aa8fe19 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -94,6 +94,7 @@ ('c:func', 'sprintf'), ('c:func', 'stat'), ('c:func', 'system'), + ('c:func', 'time'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index fcf711efe0eb7d..474359a5cd98bd 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -41,7 +41,7 @@ You load libraries by accessing them as attributes of these objects. *cdll* loads libraries which export functions using the standard ``cdecl`` calling convention, while *windll* libraries call functions using the ``stdcall`` calling convention. *oledll* also uses the ``stdcall`` calling convention, and -assumes the functions return a Windows :c:type:`HRESULT` error code. The error +assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -477,7 +477,7 @@ Return types By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`restype` attribute of the +return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the function object. The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` @@ -495,7 +495,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the :func:`strchr` function, which expects +Here is a more advanced example, it uses the :func:`!strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -528,7 +528,7 @@ single character Python bytes object into a C char: >>> You can also use a callable Python object (a function or a class for example) as -the :attr:`restype` attribute, if the foreign function returns an integer. The +the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The callable will be called with the *integer* the C function returns, and the result of this call will be used as the result of your function call. This is useful to check for error return values and automatically raise an exception:: @@ -556,7 +556,8 @@ get the string representation of an error code, and *returns* an exception. :func:`GetLastError` to retrieve it. Please note that a much more powerful error checking mechanism is available -through the :attr:`errcheck` attribute; see the reference manual for details. +through the :attr:`~_FuncPtr.errcheck` attribute; +see the reference manual for details. .. _ctypes-passing-pointers: @@ -594,7 +595,7 @@ Structures and unions Structures and unions must derive from the :class:`Structure` and :class:`Union` base classes which are defined in the :mod:`ctypes` module. Each subclass must -define a :attr:`_fields_` attribute. :attr:`_fields_` must be a list of +define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other @@ -666,9 +667,9 @@ Structure/union alignment and byte order By default, Structure and Union fields are aligned in the same way the C compiler does it. It is possible to override this behavior by specifying a -:attr:`_pack_` class attribute in the subclass definition. This must be set to a -positive integer and specifies the maximum alignment for the fields. This is -what ``#pragma pack(n)`` also does in MSVC. +:attr:`~Structure._pack_` class attribute in the subclass definition. +This must be set to a positive integer and specifies the maximum alignment for the fields. +This is what ``#pragma pack(n)`` also does in MSVC. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -684,7 +685,7 @@ Bit fields in structures and unions It is possible to create structures and unions containing bit fields. Bit fields are only possible for integer fields, the bit width is specified as the third -item in the :attr:`_fields_` tuples:: +item in the :attr:`~Structure._fields_` tuples:: >>> class Int(Structure): ... _fields_ = [("first_16", c_int, 16), @@ -876,7 +877,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -952,8 +953,8 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the :attr:`_fields_` -attribute later, after the class statement:: +In :mod:`ctypes`, we can define the ``cell`` class and set the +:attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * >>> class cell(Structure): @@ -1003,8 +1004,8 @@ argument, and the callback functions expected argument types as the remaining arguments. I will present an example here which uses the standard C library's -:c:func:`qsort` function, that is used to sort items with the help of a callback -function. :c:func:`qsort` will be used to sort an array of integers:: +:c:func:`!qsort` function, that is used to sort items with the help of a callback +function. :c:func:`!qsort` will be used to sort an array of integers:: >>> IntArray5 = c_int * 5 >>> ia = IntArray5(5, 1, 7, 33, 99) @@ -1012,7 +1013,7 @@ function. :c:func:`qsort` will be used to sort an array of integers:: >>> qsort.restype = None >>> -:func:`qsort` must be called with a pointer to the data to sort, the number of +:func:`!qsort` must be called with a pointer to the data to sort, the number of items in the data array, the size of one item, and a pointer to the comparison function, the callback. The callback will then be called with two pointers to items, and it must return a negative integer if the first item is smaller than @@ -1104,7 +1105,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`in_dll` class methods of +:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1294,13 +1295,13 @@ Finding shared libraries When programming in a compiled language, shared libraries are accessed when compiling/linking a program, and when the program is run. -The purpose of the :func:`find_library` function is to locate a library in a way +The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way similar to what the compiler or runtime loader does (on platforms with several versions of a shared library the most recent should be loaded), while the ctypes library loaders act like when a program is run, and call the runtime loader directly. -The :mod:`ctypes.util` module provides a function which can help to determine +The :mod:`!ctypes.util` module provides a function which can help to determine the library to load. @@ -1315,7 +1316,7 @@ the library to load. The exact functionality is system dependent. -On Linux, :func:`find_library` tries to run external programs +On Linux, :func:`~ctypes.util.find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. It returns the filename of the library file. @@ -1334,7 +1335,7 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`find_library` tries several predefined naming schemes and paths +On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths to locate the library, and returns a full pathname if successful:: >>> from ctypes.util import find_library @@ -1348,13 +1349,13 @@ to locate the library, and returns a full pathname if successful:: '/System/Library/Frameworks/AGL.framework/AGL' >>> -On Windows, :func:`find_library` searches along the system search path, and +On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`find_library` to locate the library at runtime. +module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. .. _ctypes-loading-shared-libraries: @@ -1439,9 +1440,9 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` -function is used to load the library into the process, and to get a handle to -it. +parameter, otherwise the underlying platforms :c:func:`!dlopen` or +:c:func:`!LoadLibrary` function is used to load the library into +the process, and to get a handle to it. The *mode* parameter can be used to specify how the library is loaded. For details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is @@ -1461,7 +1462,7 @@ to a new value and returns the former value. The *use_last_error* parameter, when set to true, enables the same mechanism for the Windows error code which is managed by the :func:`GetLastError` and -:func:`SetLastError` Windows API functions; :func:`ctypes.get_last_error` and +:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and :func:`ctypes.set_last_error` are used to request and change the ctypes private copy of the windows error code. @@ -1533,7 +1534,7 @@ attribute of the loader instance. Class which loads shared libraries. *dlltype* should be one of the :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - :meth:`__getattr__` has special behavior: It allows loading a shared library by + :meth:`!__getattr__` has special behavior: It allows loading a shared library by accessing it as attribute of a library loader instance. The result is cached, so repeated attribute accesses return the same library each time. @@ -1578,7 +1579,7 @@ object is available: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`restype` attribute to use these functions. + the correct :attr:`!restype` attribute to use these functions. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader @@ -1630,7 +1631,7 @@ They are instances of a private class: the callable will be called with this integer, allowing further processing or error checking. Using this is deprecated, for more flexible post processing or error checking use a ctypes data type as - :attr:`restype` and assign a callable to the :attr:`errcheck` attribute. + :attr:`!restype` and assign a callable to the :attr:`errcheck` attribute. .. attribute:: argtypes @@ -1662,7 +1663,7 @@ They are instances of a private class: :module: *result* is what the foreign function returns, as specified by the - :attr:`restype` attribute. + :attr:`!restype` attribute. *func* is the foreign function object itself, this allows reusing the same callable object to check or post process the results of several @@ -1772,7 +1773,7 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`~_FuncPtr.argtypes` tuple. + are specified in the :attr:`!argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. @@ -1847,7 +1848,7 @@ value if there is a single one, or a tuple containing the output parameter values when there are more than one, so the GetWindowRect function now returns a RECT instance, when called. -Output parameters can be combined with the :attr:`errcheck` protocol to do +Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do further output processing and error checking. The win32 ``GetWindowRect`` api function returns a ``BOOL`` to signal success or failure, so this function could do the error checking, and raises an exception when the api call failed:: @@ -1860,7 +1861,7 @@ do the error checking, and raises an exception when the api call failed:: >>> GetWindowRect.errcheck = errcheck >>> -If the :attr:`errcheck` function returns the argument tuple it receives +If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives unchanged, :mod:`ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them @@ -2010,7 +2011,7 @@ Utility functions .. function:: get_last_error() Windows only: returns the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread. + :data:`!LastError` variable in the calling thread. .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error @@ -2063,7 +2064,7 @@ Utility functions .. function:: set_last_error(value) Windows only: set the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread to *value* and return the + :data:`!LastError` variable in the calling thread to *value* and return the previous value. .. audit-event:: ctypes.set_last_error error ctypes.set_last_error @@ -2225,13 +2226,13 @@ Fundamental data types Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently converted to native Python types. In other words, if a foreign function has a -:attr:`restype` of :class:`c_char_p`, you will always receive a Python bytes +:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes object, *not* a :class:`c_char_p` instance. .. XXX above is false, it actually returns a Unicode string Subclasses of fundamental data types do *not* inherit this behavior. So, if a -foreign functions :attr:`restype` is a subclass of :class:`c_void_p`, you will +foreign functions :attr:`!restype` is a subclass of :class:`c_void_p`, you will receive an instance of this subclass from the function call. Of course, you can get the value of the pointer by accessing the ``value`` attribute. @@ -2430,7 +2431,7 @@ These are the fundamental ctypes data types: .. class:: HRESULT - Windows only: Represents a :c:type:`HRESULT` value, which contains success or + Windows only: Represents a :c:type:`!HRESULT` value, which contains success or error information for a function or method call. @@ -2439,9 +2440,9 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`PyObject *` datatype. Calling this without an argument creates a ``NULL`` :c:expr:`PyObject *` pointer. -The :mod:`ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`HWND`, :c:type:`WPARAM`, or :c:type:`DWORD`. Some -useful structures like :c:type:`MSG` or :c:type:`RECT` are also defined. +The :mod:`!ctypes.wintypes` module provides quite some other Windows specific +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. .. _ctypes-structured-data-types: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index df7dcd607489c8..21b350c4134fd7 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -59,7 +59,6 @@ Doc/library/configparser.rst Doc/library/contextlib.rst Doc/library/copy.rst Doc/library/csv.rst -Doc/library/ctypes.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst From c399b5e1a56f7c659fc92edc20bc4bf522bdeffd Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 7 Aug 2023 14:26:49 +0100 Subject: [PATCH 054/598] gh-107713: Reduce usage of mocks in `test_clinic.py` (#107714) --- Lib/test/test_clinic.py | 106 +++++++--------------------------------- Tools/clinic/clinic.py | 3 ++ 2 files changed, 22 insertions(+), 87 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f594e39a90546a..d13d8623f8093b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -7,7 +7,6 @@ from test.support.os_helper import TESTFN, unlink from textwrap import dedent from unittest import TestCase -import collections import contextlib import inspect import os.path @@ -21,6 +20,13 @@ from clinic import DSLParser +def _make_clinic(*, filename='clinic_tests'): + clang = clinic.CLanguage(None) + c = clinic.Clinic(clang, filename=filename) + c.block_parser = clinic.BlockParser('', clang) + return c + + def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): """Helper for the parser tests. @@ -41,81 +47,6 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): tc.assertEqual(cm.exception.lineno, lineno) -class FakeConverter: - def __init__(self, name, args): - self.name = name - self.args = args - - -class FakeConverterFactory: - def __init__(self, name): - self.name = name - - def __call__(self, name, default, **kwargs): - return FakeConverter(self.name, kwargs) - - -class FakeConvertersDict: - def __init__(self): - self.used_converters = {} - - def get(self, name, default): - return self.used_converters.setdefault(name, FakeConverterFactory(name)) - -c = clinic.Clinic(language='C', filename = "file") - -class FakeClinic: - def __init__(self): - self.converters = FakeConvertersDict() - self.legacy_converters = FakeConvertersDict() - self.language = clinic.CLanguage(None) - self.filename = "clinic_tests" - self.destination_buffers = {} - self.block_parser = clinic.BlockParser('', self.language) - self.modules = collections.OrderedDict() - self.classes = collections.OrderedDict() - clinic.clinic = self - self.name = "FakeClinic" - self.line_prefix = self.line_suffix = '' - self.destinations = {} - self.add_destination("block", "buffer") - self.add_destination("file", "buffer") - self.add_destination("suppress", "suppress") - d = self.destinations.get - self.field_destinations = collections.OrderedDict(( - ('docstring_prototype', d('suppress')), - ('docstring_definition', d('block')), - ('methoddef_define', d('block')), - ('impl_prototype', d('block')), - ('parser_prototype', d('suppress')), - ('parser_definition', d('block')), - ('impl_definition', d('block')), - )) - self.functions = [] - - def get_destination(self, name): - d = self.destinations.get(name) - if not d: - sys.exit("Destination does not exist: " + repr(name)) - return d - - def add_destination(self, name, type, *args): - if name in self.destinations: - sys.exit("Destination already exists: " + repr(name)) - self.destinations[name] = clinic.Destination(name, type, self, *args) - - def is_directive(self, name): - return name == "module" - - def directive(self, name, args): - self.called_directives[name] = args - - _module_and_class = clinic.Clinic._module_and_class - - def __repr__(self): - return "" - - class ClinicWholeFileTest(TestCase): maxDiff = None @@ -124,7 +55,7 @@ def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): filename=filename, lineno=lineno) def setUp(self): - self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") + self.clinic = _make_clinic(filename="test.c") def test_eol(self): # regression test: @@ -848,7 +779,7 @@ def test_clinic_1(self): class ClinicParserTest(TestCase): def parse(self, text): - c = FakeClinic() + c = _make_clinic() parser = DSLParser(c) block = clinic.Block(text) parser.parse(block) @@ -872,7 +803,7 @@ def checkDocstring(self, fn, expected): dedent(expected).strip()) def test_trivial(self): - parser = DSLParser(FakeClinic()) + parser = DSLParser(_make_clinic()) block = clinic.Block(""" module os os.access @@ -1119,7 +1050,7 @@ def test_cloning_nonexistent_function_correctly_fails(self): with support.captured_stderr() as stderr: self.expect_failure(block, err, lineno=0) expected_debug_print = dedent("""\ - cls=None, module=, existing='fooooooooooooooooo' + cls=None, module=, existing='fooooooooooooooooo' (cls or module).functions=[] """) stderr = stderr.getvalue() @@ -1740,8 +1671,7 @@ def test_indent_stack_illegal_outdent(self): self.expect_failure(block, err) def test_directive(self): - c = FakeClinic() - parser = DSLParser(c) + parser = DSLParser(_make_clinic()) parser.flag = False parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) block = clinic.Block("setflag") @@ -3147,22 +3077,24 @@ def test_Block_repr(self): self.assertEqual(repr(block3), expected_repr_3) def test_Destination_repr(self): + c = _make_clinic() + destination = clinic.Destination( - "foo", type="file", clinic=FakeClinic(), args=("eggs",) + "foo", type="file", clinic=c, args=("eggs",) ) self.assertEqual( repr(destination), "" ) - destination2 = clinic.Destination("bar", type="buffer", clinic=FakeClinic()) + destination2 = clinic.Destination("bar", type="buffer", clinic=c) self.assertEqual(repr(destination2), "") def test_Module_repr(self): - module = clinic.Module("foo", FakeClinic()) + module = clinic.Module("foo", _make_clinic()) self.assertRegex(repr(module), r"") def test_Class_repr(self): - cls = clinic.Class("foo", FakeClinic(), None, 'some_typedef', 'some_type_object') + cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): @@ -3176,7 +3108,7 @@ def test_FunctionKind_repr(self): def test_Function_and_Parameter_reprs(self): function = clinic.Function( name='foo', - module=FakeClinic(), + module=_make_clinic(), cls=None, c_basename=None, full_name='foofoo', diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 4dfe90b314f543..2bed98c23674e0 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2424,6 +2424,9 @@ def _module_and_class( return module, cls + def __repr__(self) -> str: + return "" + def parse_file( filename: str, From 8fcee6b2795a504f1bd39118759e5ce44183f689 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 7 Aug 2023 16:52:36 +0300 Subject: [PATCH 055/598] gh-107710: Speed up `logging.getHandlerNames` function (#107711) --- Lib/logging/__init__.py | 3 +-- .../Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 46e86cb87ecfcb..527fc5c631730a 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -916,8 +916,7 @@ def getHandlerNames(): """ Return all known handler names as an immutable set. """ - result = set(_handlers.keys()) - return frozenset(result) + return frozenset(_handlers) class Handler(Filterer): diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst new file mode 100644 index 00000000000000..70f8b58e7ff5f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst @@ -0,0 +1 @@ +Speed up :func:`logging.getHandlerNames`. From 835e38891535649321230f94121410244c583966 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 7 Aug 2023 16:33:52 +0200 Subject: [PATCH 056/598] gh-95065: Argument Clinic: Pretty-print long C strings in generated code (#107712) Co-authored-by: Serhiy Storchaka --- Lib/test/clinic.test.c | 223 +++++++++++++++++++++++++++++++---------- Tools/clinic/clinic.py | 37 +++++-- 2 files changed, 199 insertions(+), 61 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 321ac69273189f..c5c37b7e9e2168 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5521,17 +5521,31 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * PyObject *b = Py_None; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1_optional' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1_optional' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1_optional' to be " \ + "keyword-only." # endif #endif if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1_optional() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to " + "test_deprecate_positional_pos1_len1_optional() is deprecated. " + "Parameter 'b' will become a keyword-only parameter in Python " + "3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); @@ -5553,7 +5567,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * static PyObject * test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=20bdea6a2960ddf3 input=89099f3dacd757da]*/ +/*[clinic end generated code: output=09a6edec1ddcd469 input=89099f3dacd757da]*/ /*[clinic input] @@ -5609,17 +5623,27 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ PyObject *b; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1' to be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1' to be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1' to be keyword-only." # endif #endif if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to " + "test_deprecate_positional_pos1_len1() is deprecated. Parameter " + "'b' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -5637,7 +5661,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=22c70f8b36085758 input=1702bbab1e9b3b99]*/ +/*[clinic end generated code: output=52a2618293df747d input=1702bbab1e9b3b99]*/ /*[clinic input] @@ -5699,17 +5723,31 @@ test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const * PyObject *d; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ + "keyword-only." # endif #endif if (nargs > 1 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to test_deprecate_positional_pos1_len2_with_kwd() is deprecated. Parameters 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "test_deprecate_positional_pos1_len2_with_kwd() is deprecated. " + "Parameters 'b' and 'c' will become keyword-only parameters in " + "Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); @@ -5730,7 +5768,7 @@ static PyObject * test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=79c5f04220a1f3aa input=28cdb885f6c34eab]*/ +/*[clinic end generated code: output=550aabea548589b4 input=28cdb885f6c34eab]*/ /*[clinic input] @@ -5783,17 +5821,27 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ PyObject *a; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'test_deprecate_positional_pos0_len1' to be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'test_deprecate_positional_pos0_len1' to be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'test_deprecate_positional_pos0_len1' to be keyword-only." # endif #endif if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len1() is deprecated. Parameter 'a' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "test_deprecate_positional_pos0_len1() is deprecated. Parameter " + "'a' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -5809,7 +5857,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=1b7f23b9ffca431b input=678206db25c0652c]*/ +/*[clinic end generated code: output=66c63ec8d6903bde input=678206db25c0652c]*/ /*[clinic input] @@ -5865,17 +5913,30 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ PyObject *b; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'test_deprecate_positional_pos0_len2' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'test_deprecate_positional_pos0_len2' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'test_deprecate_positional_pos0_len2' to be " \ + "keyword-only." # endif #endif if (nargs > 0 && nargs <= 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len2() is deprecated. Parameters 'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "test_deprecate_positional_pos0_len2() is deprecated. Parameters " + "'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -5893,7 +5954,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=31b494f2dcc016af input=fae0d0b1d480c939]*/ +/*[clinic end generated code: output=6b6df40aaf751b2e input=fae0d0b1d480c939]*/ /*[clinic input] @@ -5958,17 +6019,34 @@ test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *e; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of " \ + "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of " \ + "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of " \ + "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ + "keyword-only." # endif #endif if (nargs > 0 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated. Parameters 'a', 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "test_deprecate_positional_pos0_len3_with_kwdonly() is " + "deprecated. Parameters 'a', 'b' and 'c' will become keyword-only" + " parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); @@ -5991,7 +6069,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, PyObject *b, PyObject *c, PyObject *e) -/*[clinic end generated code: output=96978e786acfbc7b input=1b0121770c0c52e0]*/ +/*[clinic end generated code: output=5c936993846d01a3 input=1b0121770c0c52e0]*/ /*[clinic input] @@ -6049,17 +6127,27 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ PyObject *c; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ + " 'test_deprecate_positional_pos2_len1' to be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ + " 'test_deprecate_positional_pos2_len1' to be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ + " 'test_deprecate_positional_pos2_len1' to be keyword-only." # endif #endif if (nargs == 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 3 positional arguments to test_deprecate_positional_pos2_len1() is deprecated. Parameter 'c' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 3 positional arguments to " + "test_deprecate_positional_pos2_len1() is deprecated. Parameter " + "'c' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); @@ -6078,7 +6166,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=ceadd05f11f7f491 input=e1d129689e69ec7c]*/ +/*[clinic end generated code: output=2641e037296e3b61 input=e1d129689e69ec7c]*/ /*[clinic input] @@ -6139,17 +6227,30 @@ test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ PyObject *d; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len2' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len2' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len2' to be " \ + "keyword-only." # endif #endif if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len2() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "test_deprecate_positional_pos2_len2() is deprecated. Parameters " + "'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); @@ -6170,7 +6271,7 @@ static PyObject * test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=5693682e3fa1188b input=0d53533463a12792]*/ +/*[clinic end generated code: output=4a9068ef8fee61f6 input=0d53533463a12792]*/ /*[clinic input] @@ -6238,17 +6339,31 @@ test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *e; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ + "be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ + "be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ + "be keyword-only." # endif #endif if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "test_deprecate_positional_pos2_len3_with_kwdonly() is " + "deprecated. Parameters 'c' and 'd' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); @@ -6273,4 +6388,4 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=00d436de747a00f3 input=154fd450448d8935]*/ +/*[clinic end generated code: output=1154c2e3e798948c input=154fd450448d8935]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2bed98c23674e0..3a8431cb67efea 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -221,6 +221,20 @@ def c_repr(s: str) -> str: return '"' + s + '"' +def wrapped_c_string_literal( + text: str, + *, + width: int = 72, + suffix: str = '', + initial_indent: int = 0, + subsequent_indent: int = 4 +) -> str: + wrapped = textwrap.wrap(text, width=width, replace_whitespace=False, + drop_whitespace=False, break_on_hyphens=False) + separator = '"' + suffix + '\n' + subsequent_indent * ' ' + '"' + return initial_indent * ' ' + '"' + separator.join(wrapped) + '"' + + is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match def is_legal_py_identifier(s: str) -> bool: @@ -837,17 +851,22 @@ class CLanguage(Language): """) DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error "{cpp_message}" + # error \ + {cpp_message} #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 # ifdef _MSC_VER - # pragma message ("{cpp_message}") + # pragma message ( \ + {cpp_message}) # else - # warning "{cpp_message}" + # warning \ + {cpp_message} # endif #endif if ({condition}) {{{{ - if (PyErr_WarnEx(PyExc_DeprecationWarning, "{depr_message}", 1)) {{{{ - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + {depr_message}, 1)) + {{{{ + goto exit; }}}} }}}} """ @@ -899,6 +918,7 @@ def deprecate_positional_use( f"In {source}, update parameter(s) {pstr} in the clinic " f"input of {func.full_name!r} to be keyword-only." ) + # Format the deprecation message. if first_pos == 0: preamble = "Passing positional arguments to " @@ -926,8 +946,11 @@ def deprecate_positional_use( condition=condition, major=major, minor=minor, - cpp_message=cpp_message, - depr_message=depr_message, + cpp_message=wrapped_c_string_literal(cpp_message, suffix=" \\", + width=64, + subsequent_indent=16), + depr_message=wrapped_c_string_literal(depr_message, width=64, + subsequent_indent=20), ) return normalize_snippet(code, indent=4) From 50e3cc9748eb2103eb7ed6cc5a74d177df3cfb13 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 17:38:55 +0300 Subject: [PATCH 057/598] gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) Passing a callable object as an option value to a Tkinter image now raises the expected TclError instead of an AttributeError. --- Lib/test/test_tkinter/test_images.py | 16 ++++++++++++++++ Lib/tkinter/__init__.py | 4 ---- ...023-08-06-15-29-00.gh-issue-100814.h195gW.rst | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index b6ba1c22dcbaca..317b0a5c8f4a30 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -144,6 +144,14 @@ def test_configure_foreground(self): self.assertEqual(image['foreground'], '-foreground {} {} #000000 yellow') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.BitmapImage('::img::test', master=self.root, spam=print) + image = tkinter.BitmapImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + class PhotoImageTest(AbstractTkTest, unittest.TestCase): @@ -274,6 +282,14 @@ def test_configure_palette(self): image.configure(palette='3/4/2') self.assertEqual(image['palette'], '3/4/2') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.PhotoImage('::img::test', master=self.root, spam=print) + image = tkinter.PhotoImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + def test_blank(self): image = self.create() image.blank() diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c675c511e04533..c59f8d11e8a9da 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4069,8 +4069,6 @@ def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): elif kw: cnf = kw options = () for k, v in cnf.items(): - if callable(v): - v = self._register(v) options = options + ('-'+k, v) self.tk.call(('image', 'create', imgtype, name,) + options) self.name = name @@ -4097,8 +4095,6 @@ def configure(self, **kw): for k, v in _cnfmerge(kw).items(): if v is not None: if k[-1] == '_': k = k[:-1] - if callable(v): - v = self._register(v) res = res + ('-'+k, v) self.tk.call((self.name, 'config') + res) diff --git a/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst new file mode 100644 index 00000000000000..86cb7bf79f3078 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst @@ -0,0 +1,2 @@ +Passing a callable object as an option value to a Tkinter image now raises +the expected TclError instead of an AttributeError. From ed64204716035db58c7e11b78182596aa2d97176 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 18:09:56 +0300 Subject: [PATCH 058/598] gh-106566: Optimize (?!) in regular expressions (GH-106567) --- Lib/re/_parser.py | 4 +++- Lib/test/test_re.py | 3 +++ .../Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index 22d10ab6e31d37..d00b7e67d55958 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -773,8 +773,10 @@ def _parse(source, state, verbose, nested, first=False): source.tell() - start) if char == "=": subpatternappend((ASSERT, (dir, p))) - else: + elif p: subpatternappend((ASSERT_NOT, (dir, p))) + else: + subpatternappend((FAILURE, ())) continue elif char == "(": diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a6f5af17d7d51b..a565cbe1a8d810 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2362,6 +2362,9 @@ def test_regression_gh94675(self): p.terminate() p.join() + def test_fail(self): + self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + def get_debug_out(pat): with captured_stdout() as out: diff --git a/Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst b/Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst new file mode 100644 index 00000000000000..3b88dc79183876 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst @@ -0,0 +1 @@ +Optimize ``(?!)`` (pattern which alwais fails) in regular expressions. From 85793278793708ad6b7132a54ac9fb4b2c5bcac1 Mon Sep 17 00:00:00 2001 From: Gertjan van Zwieten Date: Mon, 7 Aug 2023 18:24:02 +0300 Subject: [PATCH 059/598] gh-107715: Escape class name in regular expression (GH-107716) This patch escapes the class name before embedding it in the regular expression for `pat` in `doctest.DocTestFinder._find_lineno`. While class names do not ordinarily contain special characters, it is possible to encounter these when a class is created dynamically. Escaping the name will correctly return `None` in this scenario, rather than potentially matching a different class or raising `re.error` depending on the symbols used. --- Lib/doctest.py | 2 +- .../next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index 2776d74bf9b586..a63df46a112e64 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1110,7 +1110,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst new file mode 100644 index 00000000000000..cd2a5d0d5324a2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst @@ -0,0 +1 @@ +Fix `doctest.DocTestFinder.find` in presence of class names with special characters. Patch by Gertjan van Zwieten. From 16c9415fba4972743f1944ebc44946e475e68bc4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 18:51:43 +0300 Subject: [PATCH 060/598] gh-107178: Add the C API tests for the Abstract Objects Layer (GH-107179) Cover all the Mapping Protocol, almost all the Sequence Protocol (except PySequence_Fast) and a part of the Object Protocol. Move existing tests to Lib/test/test_capi/test_abstract.py and Modules/_testcapi/abstract.c. Add also tests for PyDict C API. --- Lib/test/test_bytes.py | 2 +- Lib/test/test_capi/test_abstract.py | 722 ++++++++++++++++++ Lib/test/test_capi/test_dict.py | 413 ++++++++++ Lib/test/test_capi/test_misc.py | 131 ---- Lib/test/test_class.py | 4 +- ...-07-24-16-56-59.gh-issue-107178.Gq1usE.rst | 2 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/abstract.c | 640 ++++++++++++++++ Modules/_testcapi/dict.c | 376 +++++++++ Modules/_testcapi/parts.h | 2 + Modules/_testcapimodule.c | 169 +--- PCbuild/_testcapi.vcxproj | 2 + PCbuild/_testcapi.vcxproj.filters | 6 + 13 files changed, 2174 insertions(+), 297 deletions(-) create mode 100644 Lib/test/test_capi/test_abstract.py create mode 100644 Lib/test/test_capi/test_dict.py create mode 100644 Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst create mode 100644 Modules/_testcapi/abstract.c create mode 100644 Modules/_testcapi/dict.c diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 7c62b722059d12..afd506f07520d8 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1354,7 +1354,7 @@ def do_tests(setitem): except ValueError: pass try: - setitem(b, 0, None) + setitem(b, 0, object()) self.fail("Didn't raise TypeError") except TypeError: pass diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py new file mode 100644 index 00000000000000..3f51e5b28104d9 --- /dev/null +++ b/Lib/test/test_capi/test_abstract.py @@ -0,0 +1,722 @@ +import unittest +import sys +from collections import OrderedDict +from test import support +from test.support import import_helper +import _testcapi + + +NULL = None + +class TestObject: + @property + def evil(self): + raise RuntimeError('do not get evil') + @evil.setter + def evil(self, value): + raise RuntimeError('do not set evil') + @evil.deleter + def evil(self): + raise RuntimeError('do not del evil') + +class ProxyGetItem: + def __init__(self, obj): + self.obj = obj + def __getitem__(self, key): + return self.obj[key] + +class ProxySetItem: + def __init__(self, obj): + self.obj = obj + def __setitem__(self, key, value): + self.obj[key] = value + +class ProxyDelItem: + def __init__(self, obj): + self.obj = obj + def __delitem__(self, key): + del self.obj[key] + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_object_getattr(self): + xgetattr = _testcapi.object_getattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(xgetattr(obj, 'a'), 11) + self.assertRaises(AttributeError, xgetattr, obj, 'b') + self.assertEqual(xgetattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, xgetattr, obj, 'evil') + self.assertRaises(TypeError, xgetattr, obj, 1) + # CRASHES xgetattr(obj, NULL) + # CRASHES xgetattr(NULL, 'a') + + def test_object_getattrstring(self): + getattrstring = _testcapi.object_getattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getattrstring(obj, b'a'), 11) + self.assertRaises(AttributeError, getattrstring, obj, b'b') + self.assertEqual(getattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getattrstring, obj, b'\xff') + # CRASHES getattrstring(obj, NULL) + # CRASHES getattrstring(NULL, b'a') + + def test_object_getoptionalattr(self): + getoptionalattr = _testcapi.object_getoptionalattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getoptionalattr(obj, 'a'), 11) + self.assertIs(getoptionalattr(obj, 'b'), AttributeError) + self.assertEqual(getoptionalattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, getoptionalattr, obj, 'evil') + self.assertRaises(TypeError, getoptionalattr, obj, 1) + # CRASHES getoptionalattr(obj, NULL) + # CRASHES getoptionalattr(NULL, 'a') + + def test_object_getoptionalattrstring(self): + getoptionalattrstring = _testcapi.object_getoptionalattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getoptionalattrstring(obj, b'a'), 11) + self.assertIs(getoptionalattrstring(obj, b'b'), AttributeError) + self.assertEqual(getoptionalattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getoptionalattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getoptionalattrstring, obj, b'\xff') + # CRASHES getoptionalattrstring(obj, NULL) + # CRASHES getoptionalattrstring(NULL, b'a') + + def test_object_hasattr(self): + xhasattr = _testcapi.object_hasattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(xhasattr(obj, 'a')) + self.assertFalse(xhasattr(obj, 'b')) + self.assertTrue(xhasattr(obj, '\U0001f40d')) + + self.assertFalse(xhasattr(obj, 'evil')) + self.assertFalse(xhasattr(obj, 1)) + # CRASHES xhasattr(obj, NULL) + # CRASHES xhasattr(NULL, 'a') + + def test_object_hasattrstring(self): + hasattrstring = _testcapi.object_hasattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(hasattrstring(obj, b'a')) + self.assertFalse(hasattrstring(obj, b'b')) + self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) + + self.assertFalse(hasattrstring(obj, b'evil')) + self.assertFalse(hasattrstring(obj, b'\xff')) + # CRASHES hasattrstring(obj, NULL) + # CRASHES hasattrstring(NULL, b'a') + + def test_object_setattr(self): + xsetattr = _testcapi.object_setattr + obj = TestObject() + xsetattr(obj, 'a', 5) + self.assertEqual(obj.a, 5) + xsetattr(obj, '\U0001f40d', 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttr(obj, attr_name, NULL) removes the attribute + xsetattr(obj, 'a', NULL) + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, xsetattr, obj, 'b', NULL) + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) + + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') + self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) + self.assertRaises(TypeError, xsetattr, obj, 1, 5) + # CRASHES xsetattr(obj, NULL, 5) + # CRASHES xsetattr(NULL, 'a', 5) + + def test_object_setattrstring(self): + setattrstring = _testcapi.object_setattrstring + obj = TestObject() + setattrstring(obj, b'a', 5) + self.assertEqual(obj.a, 5) + setattrstring(obj, '\U0001f40d'.encode(), 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttrString(obj, attr_name, NULL) removes the attribute + setattrstring(obj, b'a', NULL) + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, setattrstring, obj, b'b', NULL) + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) + + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') + self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) + self.assertRaises(TypeError, setattrstring, obj, 1, 5) + self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) + # CRASHES setattrstring(obj, NULL, 5) + # CRASHES setattrstring(NULL, b'a', 5) + + def test_object_delattr(self): + xdelattr = _testcapi.object_delattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + xdelattr(obj, 'a') + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, xdelattr, obj, 'b') + xdelattr(obj, '\U0001f40d') + self.assertFalse(hasattr(obj, '\U0001f40d')) + + self.assertRaises(AttributeError, xdelattr, 42, 'numerator') + self.assertRaises(RuntimeError, xdelattr, obj, 'evil') + self.assertRaises(TypeError, xdelattr, obj, 1) + # CRASHES xdelattr(obj, NULL) + # CRASHES xdelattr(NULL, 'a') + + def test_object_delattrstring(self): + delattrstring = _testcapi.object_delattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + delattrstring(obj, b'a') + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, delattrstring, obj, b'b') + delattrstring(obj, '\U0001f40d'.encode()) + self.assertFalse(hasattr(obj, '\U0001f40d')) + + self.assertRaises(AttributeError, delattrstring, 42, b'numerator') + self.assertRaises(RuntimeError, delattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') + # CRASHES delattrstring(obj, NULL) + # CRASHES delattrstring(NULL, b'a') + + + def test_mapping_check(self): + check = _testcapi.mapping_check + self.assertTrue(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_mapping_size(self): + for size in _testcapi.mapping_size, _testcapi.mapping_length: + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_object_getitem(self): + getitem = _testcapi.object_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertRaises(KeyError, getitem, dct, 'b') + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertRaises(KeyError, getitem, dct2, 'b') + + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') + + self.assertRaises(TypeError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, {}, NULL) + self.assertRaises(IndexError, getitem, [], 1) + self.assertRaises(TypeError, getitem, [], 'a') + self.assertRaises(SystemError, getitem, NULL, 'a') + + def test_mapping_getitemstring(self): + getitemstring = _testcapi.mapping_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct, b'b') + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct2, b'b') + + self.assertRaises(TypeError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, {}, NULL) + self.assertRaises(TypeError, getitemstring, [], b'a') + self.assertRaises(SystemError, getitemstring, NULL, b'a') + + def test_mapping_haskey(self): + haskey = _testcapi.mapping_haskey + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskey(dct, 'a')) + self.assertFalse(haskey(dct, 'b')) + self.assertTrue(haskey(dct, '\U0001f40d')) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskey(dct2, 'a')) + self.assertFalse(haskey(dct2, 'b')) + + self.assertTrue(haskey(['a', 'b', 'c'], 1)) + + self.assertFalse(haskey(42, 'a')) + self.assertFalse(haskey({}, [])) # unhashable + self.assertFalse(haskey({}, NULL)) + self.assertFalse(haskey([], 1)) + self.assertFalse(haskey([], 'a')) + self.assertFalse(haskey(NULL, 'a')) + + def test_mapping_haskeystring(self): + haskeystring = _testcapi.mapping_haskeystring + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskeystring(dct, b'a')) + self.assertFalse(haskeystring(dct, b'b')) + self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskeystring(dct2, b'a')) + self.assertFalse(haskeystring(dct2, b'b')) + + self.assertFalse(haskeystring(42, b'a')) + self.assertFalse(haskeystring({}, b'\xff')) + self.assertFalse(haskeystring({}, NULL)) + self.assertFalse(haskeystring([], b'a')) + self.assertFalse(haskeystring(NULL, b'a')) + + def test_object_setitem(self): + setitem = _testcapi.object_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitem(dct2, 'a', 5) + self.assertEqual(dct, {'a': 5}) + + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + + self.assertRaises(TypeError, setitem, 42, 'a', 5) + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, {}, NULL, 5) + self.assertRaises(SystemError, setitem, {}, 'a', NULL) + self.assertRaises(IndexError, setitem, [], 1, 5) + self.assertRaises(TypeError, setitem, [], 'a', 5) + self.assertRaises(TypeError, setitem, (), 1, 5) + self.assertRaises(SystemError, setitem, NULL, 'a', 5) + + def test_mapping_setitemstring(self): + setitemstring = _testcapi.mapping_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitemstring(dct2, b'a', 5) + self.assertEqual(dct, {'a': 5}) + + self.assertRaises(TypeError, setitemstring, 42, b'a', 5) + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, {}, NULL, 5) + self.assertRaises(SystemError, setitemstring, {}, b'a', NULL) + self.assertRaises(TypeError, setitemstring, [], b'a', 5) + self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) + + def test_object_delitem(self): + for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitem(dct2, 'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + + self.assertRaises(TypeError, delitem, 42, 'a') + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, {}, NULL) + self.assertRaises(IndexError, delitem, [], 1) + self.assertRaises(TypeError, delitem, [], 'a') + self.assertRaises(SystemError, delitem, NULL, 'a') + + def test_mapping_delitemstring(self): + delitemstring = _testcapi.mapping_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitemstring(dct2, b'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(TypeError, delitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, {}, NULL) + self.assertRaises(TypeError, delitemstring, [], b'a') + self.assertRaises(SystemError, delitemstring, NULL, b'a') + + def test_mapping_keys_valuesitems(self): + class Mapping1(dict): + def keys(self): + return list(super().keys()) + def values(self): + return list(super().values()) + def items(self): + return list(super().items()) + class Mapping2(dict): + def keys(self): + return tuple(super().keys()) + def values(self): + return tuple(super().values()) + def items(self): + return tuple(super().items()) + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + + for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), + dict_obj, OrderedDict(dict_obj), + Mapping1(dict_obj), Mapping2(dict_obj)]: + self.assertListEqual(_testcapi.mapping_keys(mapping), + list(mapping.keys())) + self.assertListEqual(_testcapi.mapping_values(mapping), + list(mapping.values())) + self.assertListEqual(_testcapi.mapping_items(mapping), + list(mapping.items())) + + def test_mapping_keys_valuesitems_bad_arg(self): + self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testcapi.mapping_values, []) + self.assertRaises(AttributeError, _testcapi.mapping_items, []) + self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + + class BadMapping: + def keys(self): + return None + def values(self): + return None + def items(self): + return None + bad_mapping = BadMapping() + self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + + def test_sequence_check(self): + check = _testcapi.sequence_check + self.assertFalse(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_sequence_size(self): + for size in _testcapi.sequence_size, _testcapi.sequence_length: + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, {}) + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_sequence_getitem(self): + getitem = _testcapi.sequence_getitem + lst = ['a', 'b', 'c'] + self.assertEqual(getitem(lst, 1), 'b') + self.assertEqual(getitem(lst, -1), 'c') + self.assertRaises(IndexError, getitem, lst, 3) + + self.assertRaises(TypeError, getitem, 42, 1) + self.assertRaises(TypeError, getitem, {}, 1) + self.assertRaises(SystemError, getitem, NULL, 1) + + def test_sequence_concat(self): + concat = _testcapi.sequence_concat + self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) + self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, concat, [], ()) + self.assertRaises(TypeError, concat, (), []) + self.assertRaises(TypeError, concat, [], 42) + self.assertRaises(TypeError, concat, 42, []) + self.assertRaises(TypeError, concat, 42, 43) + self.assertRaises(SystemError, concat, [], NULL) + self.assertRaises(SystemError, concat, NULL, []) + + def test_sequence_repeat(self): + repeat = _testcapi.sequence_repeat + self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) + self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(repeat(['a', 'b'], 0), []) + self.assertEqual(repeat(['a', 'b'], -1), []) + + self.assertRaises(TypeError, repeat, set(), 2) + self.assertRaises(TypeError, repeat, 42, 2) + self.assertRaises(SystemError, repeat, NULL, 2) + + def test_sequence_inplaceconcat(self): + inplaceconcat = _testcapi.sequence_inplaceconcat + lst = ['a', 'b'] + res = inplaceconcat(lst, [1, 2]) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + lst = ['a', 'b'] + res = inplaceconcat(lst, (1, 2)) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + self.assertEqual(inplaceconcat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, inplaceconcat, (), []) + self.assertRaises(TypeError, inplaceconcat, [], 42) + self.assertRaises(TypeError, inplaceconcat, 42, []) + self.assertRaises(TypeError, inplaceconcat, 42, 43) + self.assertRaises(SystemError, inplaceconcat, [], NULL) + self.assertRaises(SystemError, inplaceconcat, NULL, []) + + def test_sequence_inplacerepeat(self): + inplacerepeat = _testcapi.sequence_inplacerepeat + lst = ['a', 'b'] + res = inplacerepeat(lst, 2) + self.assertEqual(res, ['a', 'b', 'a', 'b']) + self.assertIs(res, lst) + self.assertEqual(inplacerepeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(inplacerepeat(['a', 'b'], 0), []) + self.assertEqual(inplacerepeat(['a', 'b'], -1), []) + + self.assertRaises(TypeError, inplacerepeat, set(), 2) + self.assertRaises(TypeError, inplacerepeat, 42, 2) + self.assertRaises(SystemError, inplacerepeat, NULL, 2) + + def test_sequence_setitem(self): + setitem = _testcapi.sequence_setitem + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + setitem(lst, -1, 'y') + self.assertEqual(lst, ['a', 'x', 'y']) + + setitem(lst, 0, NULL) + self.assertEqual(lst, ['x', 'y']) + self.assertRaises(IndexError, setitem, lst, 3, 'x') + + self.assertRaises(TypeError, setitem, 42, 1, 'x') + self.assertRaises(TypeError, setitem, {}, 1, 'x') + self.assertRaises(SystemError, setitem, NULL, 1, 'x') + + def test_sequence_delitem(self): + delitem = _testcapi.sequence_delitem + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + delitem(lst, -1) + self.assertEqual(lst, ['a']) + self.assertRaises(IndexError, delitem, lst, 3) + + self.assertRaises(TypeError, delitem, 42, 1) + self.assertRaises(TypeError, delitem, {}, 1) + self.assertRaises(SystemError, delitem, NULL, 1) + + def test_sequence_setslice(self): + setslice = _testcapi.sequence_setslice + + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + setslice(data, 1, 3, [8, 9]) + data_copy[1:3] = [8, 9] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 8, 9, 4, 5]) + + # Custom class: + class Custom: + def __setitem__(self, index, value): + self.index = index + self.value = value + + c = Custom() + setslice(c, 0, 5, 'abc') + self.assertEqual(c.index, slice(0, 5)) + self.assertEqual(c.value, 'abc') + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, setslice, bad_seq1, 1, 3, (8, 9)) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, setslice, bad_seq2, 1, 3, 'xy') + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, setslice, object(), 1, 3, 'xy') + self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') + + data_copy = data.copy() + setslice(data_copy, 1, 3, NULL) + self.assertEqual(data_copy, [1, 4, 5]) + + def test_sequence_delslice(self): + delslice = _testcapi.sequence_delslice + + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + delslice(data, 1, 3) + del data_copy[1:3] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 4, 5]) + + # Custom class: + class Custom: + def __delitem__(self, index): + self.index = index + + c = Custom() + delslice(c, 0, 5) + self.assertEqual(c.index, slice(0, 5)) + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, delslice, bad_seq1, 1, 3) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, delslice, bad_seq2, 1, 3) + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, delslice, object(), 1, 3) + self.assertRaises(SystemError, delslice, NULL, 1, 3) + + mapping = {1: 'a', 2: 'b', 3: 'c'} + self.assertRaises(KeyError, delslice, mapping, 1, 3) + self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) + + def test_sequence_count(self): + count = _testcapi.sequence_count + + lst = ['a', 'b', 'a'] + self.assertEqual(count(lst, 'a'), 2) + self.assertEqual(count(lst, 'c'), 0) + self.assertEqual(count(iter(lst), 'a'), 2) + self.assertEqual(count(iter(lst), 'c'), 0) + self.assertEqual(count({'a': 2}, 'a'), 1) + + self.assertRaises(TypeError, count, 42, 'a') + self.assertRaises(SystemError, count, [], NULL) + self.assertRaises(SystemError, count, [1], NULL) + self.assertRaises(SystemError, count, NULL, 'a') + + def test_sequence_contains(self): + contains = _testcapi.sequence_contains + + lst = ['a', 'b', 'a'] + self.assertEqual(contains(lst, 'a'), 1) + self.assertEqual(contains(lst, 'c'), 0) + self.assertEqual(contains(iter(lst), 'a'), 1) + self.assertEqual(contains(iter(lst), 'c'), 0) + self.assertEqual(contains({'a': 2}, 'a'), 1) + + # XXX Only for empty sequences. Should be SystemError? + self.assertEqual(contains([], NULL), 0) + + self.assertRaises(TypeError, contains, 42, 'a') + self.assertRaises(SystemError, contains, [1], NULL) + # CRASHES contains({}, NULL) + # CRASHES contains(set(), NULL) + # CRASHES contains(NULL, 'a') + + def test_sequence_index(self): + index = _testcapi.sequence_index + + lst = ['a', 'b', 'a'] + self.assertEqual(index(lst, 'a'), 0) + self.assertEqual(index(lst, 'b'), 1) + self.assertRaises(ValueError, index, lst, 'c') + self.assertEqual(index(iter(lst), 'a'), 0) + self.assertEqual(index(iter(lst), 'b'), 1) + self.assertRaises(ValueError, index, iter(lst), 'c') + dct = {'a': 2, 'b': 3} + self.assertEqual(index(dct, 'a'), 0) + self.assertEqual(index(dct, 'b'), 1) + self.assertRaises(ValueError, index, dct, 'c') + + self.assertRaises(TypeError, index, 42, 'a') + self.assertRaises(SystemError, index, [], NULL) + self.assertRaises(SystemError, index, [1], NULL) + self.assertRaises(SystemError, index, NULL, 'a') + + def test_sequence_list(self): + xlist = _testcapi.sequence_list + self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) + self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) + self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) + self.assertEqual(xlist(gen()), ['a', 'b', 'c']) + + self.assertRaises(TypeError, xlist, 42) + self.assertRaises(SystemError, xlist, NULL) + + def test_sequence_tuple(self): + xtuple = _testcapi.sequence_tuple + self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) + self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) + self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) + self.assertEqual(xtuple(gen()), ('a', 'b', 'c')) + + self.assertRaises(TypeError, xtuple, 42) + self.assertRaises(SystemError, xtuple, NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py new file mode 100644 index 00000000000000..9da6efd695ecab --- /dev/null +++ b/Lib/test/test_capi/test_dict.py @@ -0,0 +1,413 @@ +import unittest +import sys +from collections import OrderedDict, UserDict +from types import MappingProxyType +from test import support +from test.support import import_helper +import _testcapi + + +NULL = None + +class DictSubclass(dict): + def __getitem__(self, key): + raise RuntimeError('do not get evil') + def __setitem__(self, key, value): + raise RuntimeError('do not set evil') + def __delitem__(self, key): + raise RuntimeError('do not del evil') + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_dict_check(self): + check = _testcapi.dict_check + self.assertTrue(check({1: 2})) + self.assertTrue(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + #self.assertFalse(check(NULL)) + + def test_dict_checkexact(self): + check = _testcapi.dict_checkexact + self.assertTrue(check({1: 2})) + self.assertFalse(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + #self.assertFalse(check(NULL)) + + def test_dict_new(self): + dict_new = _testcapi.dict_new + dct = dict_new() + self.assertEqual(dct, {}) + self.assertIs(type(dct), dict) + dct2 = dict_new() + self.assertIsNot(dct2, dct) + + def test_dictproxy_new(self): + dictproxy_new = _testcapi.dictproxy_new + for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): + proxy = dictproxy_new(dct) + self.assertIs(type(proxy), MappingProxyType) + self.assertEqual(proxy, dct) + with self.assertRaises(TypeError): + proxy[1] = 3 + self.assertEqual(proxy[1], 2) + dct[1] = 4 + self.assertEqual(proxy[1], 4) + + self.assertRaises(TypeError, dictproxy_new, []) + self.assertRaises(TypeError, dictproxy_new, 42) + # CRASHES dictproxy_new(NULL) + + def test_dict_copy(self): + copy = _testcapi.dict_copy + for dct in {1: 2}, OrderedDict({1: 2}): + dct_copy = copy(dct) + self.assertIs(type(dct_copy), dict) + self.assertEqual(dct_copy, dct) + + self.assertRaises(SystemError, copy, UserDict()) + self.assertRaises(SystemError, copy, []) + self.assertRaises(SystemError, copy, 42) + self.assertRaises(SystemError, copy, NULL) + + def test_dict_clear(self): + clear = _testcapi.dict_clear + dct = {1: 2} + clear(dct) + self.assertEqual(dct, {}) + + # NOTE: It is not safe to call it with OrderedDict. + + # Has no effect for non-dicts. + dct = UserDict({1: 2}) + clear(dct) + self.assertEqual(dct, {1: 2}) + lst = [1, 2] + clear(lst) + self.assertEqual(lst, [1, 2]) + clear(object()) + + # CRASHES? clear(NULL) + + def test_dict_size(self): + size = _testcapi.dict_size + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size(OrderedDict({1: 2})), 1) + + self.assertRaises(SystemError, size, UserDict()) + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_dict_getitem(self): + getitem = _testcapi.dict_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertIs(getitem({}, []), KeyError) # unhashable + self.assertIs(getitem(42, 'a'), KeyError) + self.assertIs(getitem([1], 0), KeyError) + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstring(self): + getitemstring = _testcapi.dict_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + self.assertIs(getitemstring({}, b'\xff'), KeyError) + self.assertIs(getitemstring(42, b'a'), KeyError) + self.assertIs(getitemstring([], b'a'), KeyError) + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemref(self): + getitem = _testcapi.dict_getitemref + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstringref(self): + getitemstring = _testcapi.dict_getitemstringref + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + self.assertRaises(SystemError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, [], b'a') + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemwitherror(self): + getitem = _testcapi.dict_getitemwitherror + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_contains(self): + contains = _testcapi.dict_contains + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(contains(dct, 'a')) + self.assertFalse(contains(dct, 'b')) + self.assertTrue(contains(dct, '\U0001f40d')) + + dct2 = DictSubclass(dct) + self.assertTrue(contains(dct2, 'a')) + self.assertFalse(contains(dct2, 'b')) + + self.assertRaises(TypeError, contains, {}, []) # unhashable + # CRASHES contains({}, NULL) + # CRASHES contains(UserDict(), 'a') + # CRASHES contains(42, 'a') + # CRASHES contains(NULL, 'a') + + def test_dict_setitem(self): + setitem = _testcapi.dict_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitem(dct2, 'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, UserDict(), 'a', 5) + self.assertRaises(SystemError, setitem, [1], 0, 5) + self.assertRaises(SystemError, setitem, 42, 'a', 5) + # CRASHES setitem({}, NULL, 5) + # CRASHES setitem({}, 'a', NULL) + # CRASHES setitem(NULL, 'a', 5) + + def test_dict_setitemstring(self): + setitemstring = _testcapi.dict_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitemstring(dct2, b'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, UserDict(), b'a', 5) + self.assertRaises(SystemError, setitemstring, 42, b'a', 5) + # CRASHES setitemstring({}, NULL, 5) + # CRASHES setitemstring({}, b'a', NULL) + # CRASHES setitemstring(NULL, b'a', 5) + + def test_dict_delitem(self): + delitem = _testcapi.dict_delitem + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitem(dct2, 'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, UserDict({'a': 1}), 'a') + self.assertRaises(SystemError, delitem, [1], 0) + self.assertRaises(SystemError, delitem, 42, 'a') + # CRASHES delitem({}, NULL) + # CRASHES delitem(NULL, 'a') + + def test_dict_delitemstring(self): + delitemstring = _testcapi.dict_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitemstring(dct2, b'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, UserDict({'a': 1}), b'a') + self.assertRaises(SystemError, delitemstring, 42, b'a') + # CRASHES delitemstring({}, NULL) + # CRASHES delitemstring(NULL, b'a') + + def test_dict_setdefault(self): + setdefault = _testcapi.dict_setdefault + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + + def test_mapping_keys_valuesitems(self): + class BadMapping(dict): + def keys(self): + return None + def values(self): + return None + def items(self): + return None + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: + self.assertListEqual(_testcapi.dict_keys(mapping), + list(dict_obj.keys())) + self.assertListEqual(_testcapi.dict_values(mapping), + list(dict_obj.values())) + self.assertListEqual(_testcapi.dict_items(mapping), + list(dict_obj.items())) + + def test_dict_keys_valuesitems_bad_arg(self): + for mapping in UserDict(), [], object(): + self.assertRaises(SystemError, _testcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testcapi.dict_values, mapping) + self.assertRaises(SystemError, _testcapi.dict_items, mapping) + + def test_dict_next(self): + dict_next = _testcapi.dict_next + self.assertIsNone(dict_next({}, 0)) + dct = {'a': 1, 'b': 2, 'c': 3} + pos = 0 + pairs = [] + while True: + res = dict_next(dct, pos) + if res is None: + break + rc, pos, key, value = res + self.assertEqual(rc, 1) + pairs.append((key, value)) + self.assertEqual(pairs, list(dct.items())) + + # CRASHES dict_next(NULL, 0) + + def test_dict_update(self): + update = _testcapi.dict_update + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + update(dct, cls2({'b': 3, 'c': 4})) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, update, {}, []) + self.assertRaises(AttributeError, update, {}, 42) + self.assertRaises(SystemError, update, UserDict(), {}) + self.assertRaises(SystemError, update, 42, {}) + self.assertRaises(SystemError, update, {}, NULL) + self.assertRaises(SystemError, update, NULL, {}) + + def test_dict_merge(self): + merge = _testcapi.dict_merge + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, merge, {}, [], 0) + self.assertRaises(AttributeError, merge, {}, 42, 0) + self.assertRaises(SystemError, merge, UserDict(), {}, 0) + self.assertRaises(SystemError, merge, 42, {}, 0) + self.assertRaises(SystemError, merge, {}, NULL, 0) + self.assertRaises(SystemError, merge, NULL, {}, 0) + + def test_dict_mergefromseq2(self): + mergefromseq2 = _testcapi.dict_mergefromseq2 + for cls1 in dict, DictSubclass: + for cls2 in list, iter: + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(ValueError, mergefromseq2, {}, [(1,)], 0) + self.assertRaises(ValueError, mergefromseq2, {}, [(1, 2, 3)], 0) + self.assertRaises(TypeError, mergefromseq2, {}, [1], 0) + self.assertRaises(TypeError, mergefromseq2, {}, 42, 0) + # CRASHES mergefromseq2(UserDict(), [], 0) + # CRASHES mergefromseq2(42, [], 0) + # CRASHES mergefromseq2({}, NULL, 0) + # CRASHES mergefromseq2(NULL, {}, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index e7cdd4be002a14..4ab16924478043 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -299,137 +299,6 @@ def test_getitem_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() - def test_mapping_keys_values_items(self): - class Mapping1(dict): - def keys(self): - return list(super().keys()) - def values(self): - return list(super().values()) - def items(self): - return list(super().items()) - class Mapping2(dict): - def keys(self): - return tuple(super().keys()) - def values(self): - return tuple(super().values()) - def items(self): - return tuple(super().items()) - dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} - - for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), - dict_obj, OrderedDict(dict_obj), - Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.get_mapping_keys(mapping), - list(mapping.keys())) - self.assertListEqual(_testcapi.get_mapping_values(mapping), - list(mapping.values())) - self.assertListEqual(_testcapi.get_mapping_items(mapping), - list(mapping.items())) - - def test_mapping_keys_values_items_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.get_mapping_keys, None) - self.assertRaises(AttributeError, _testcapi.get_mapping_values, None) - self.assertRaises(AttributeError, _testcapi.get_mapping_items, None) - - class BadMapping: - def keys(self): - return None - def values(self): - return None - def items(self): - return None - bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.get_mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.get_mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.get_mapping_items, bad_mapping) - - def test_mapping_has_key(self): - dct = {'a': 1} - self.assertTrue(_testcapi.mapping_has_key(dct, 'a')) - self.assertFalse(_testcapi.mapping_has_key(dct, 'b')) - - class SubDict(dict): - pass - - dct2 = SubDict({'a': 1}) - self.assertTrue(_testcapi.mapping_has_key(dct2, 'a')) - self.assertFalse(_testcapi.mapping_has_key(dct2, 'b')) - - def test_sequence_set_slice(self): - # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - _testcapi.sequence_set_slice(data, 1, 3, [8, 9]) - data_copy[1:3] = [8, 9] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 8, 9, 4, 5]) - - # Custom class: - class Custom: - def __setitem__(self, index, value): - self.index = index - self.value = value - - c = Custom() - _testcapi.sequence_set_slice(c, 0, 5, 'abc') - self.assertEqual(c.index, slice(0, 5)) - self.assertEqual(c.value, 'abc') - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(bad_seq1, 1, 3, (8, 9)) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(bad_seq2, 1, 3, 'xy') - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(None, 1, 3, 'xy') - - def test_sequence_del_slice(self): - # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - _testcapi.sequence_del_slice(data, 1, 3) - del data_copy[1:3] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 4, 5]) - - # Custom class: - class Custom: - def __delitem__(self, index): - self.index = index - - c = Custom() - _testcapi.sequence_del_slice(c, 0, 5) - self.assertEqual(c.index, slice(0, 5)) - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(bad_seq1, 1, 3) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(bad_seq2, 1, 3) - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(None, 1, 3) - - mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(KeyError): - _testcapi.sequence_del_slice(mapping, 1, 3) - self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) - @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'), 'need _testcapi.negative_refcount') def test_negative_refcount(self): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index bb35baea508c76..1531aad4f1f779 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -455,8 +455,8 @@ def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.hasattr_string(a, "attr"), True) - self.assertEqual(_testcapi.hasattr_string(a, "noattr"), False) + self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): diff --git a/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst b/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst new file mode 100644 index 00000000000000..dd6becf6b00130 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst @@ -0,0 +1,2 @@ +Add the C API test for functions in the Mapping Protocol, the Sequence +Protocol and some functions in the Object Protocol. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 11a022e3d2044e..689f1d42ef0eee 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -159,7 +159,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c new file mode 100644 index 00000000000000..10d7ff8d4a7bc8 --- /dev/null +++ b/Modules/_testcapi/abstract.c @@ -0,0 +1,640 @@ +#include // ptrdiff_t + +#include "parts.h" + +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_getoptionalattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + + switch (PyObject_GetOptionalAttr(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PyObject_GetOptionalAttr() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +object_getoptionalattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + + switch (PyObject_GetOptionalAttrString(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PyObject_GetOptionalAttrString() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_getoptionalitem(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + + switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +mapping_getoptionalitemstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + + switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, + {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, + {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c new file mode 100644 index 00000000000000..b1dfcf4c707da7 --- /dev/null +++ b/Modules/_testcapi/dict.c @@ -0,0 +1,376 @@ +#include // ptrdiff_t + +#include "parts.h" + +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + + +static PyObject * +dict_getitemref(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + + switch (PyDict_GetItemRef(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyMapping_GetItemRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +dict_getitemstringref(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + + switch (PyDict_GetItemStringRef(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyDict_GetItemStringRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_setdefault(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *defaultobj; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &defaultobj)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(defaultobj); + return PyDict_SetDefault(mapping, key, defaultobj); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_getitemref", dict_getitemref, METH_VARARGS}, + {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_setdefault", dict_setdefault, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index aaec0a61916948..65ebf80bcd1e95 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -26,6 +26,7 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); +int _PyTestCapi_Init_Abstract(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); int _PyTestCapi_Init_DateTime(PyObject *module); @@ -34,6 +35,7 @@ int _PyTestCapi_Init_Mem(PyObject *module); int _PyTestCapi_Init_Watchers(PyObject *module); int _PyTestCapi_Init_Long(PyObject *module); int _PyTestCapi_Init_Float(PyObject *module); +int _PyTestCapi_Init_Dict(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 2286a925e36e2c..23dd2cc9ad40f5 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1983,7 +1983,7 @@ return_result_with_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject* +static PyObject * getitem_with_error(PyObject *self, PyObject *args) { PyObject *map, *key; @@ -2060,90 +2060,6 @@ py_w_stopcode(PyObject *self, PyObject *args) #endif -static PyObject * -get_mapping_keys(PyObject* self, PyObject *obj) -{ - return PyMapping_Keys(obj); -} - -static PyObject * -get_mapping_values(PyObject* self, PyObject *obj) -{ - return PyMapping_Values(obj); -} - -static PyObject * -get_mapping_items(PyObject* self, PyObject *obj) -{ - return PyMapping_Items(obj); -} - -static PyObject * -test_mapping_has_key_string(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyObject *context = PyDict_New(); - PyObject *val = PyLong_FromLong(1); - - // Since this uses `const char*` it is easier to test this in C: - PyDict_SetItemString(context, "a", val); - if (!PyMapping_HasKeyString(context, "a")) { - PyErr_SetString(PyExc_RuntimeError, - "Existing mapping key does not exist"); - return NULL; - } - if (PyMapping_HasKeyString(context, "b")) { - PyErr_SetString(PyExc_RuntimeError, - "Missing mapping key exists"); - return NULL; - } - - Py_DECREF(val); - Py_DECREF(context); - Py_RETURN_NONE; -} - -static PyObject * -mapping_has_key(PyObject* self, PyObject *args) -{ - PyObject *context, *key; - if (!PyArg_ParseTuple(args, "OO", &context, &key)) { - return NULL; - } - return PyLong_FromLong(PyMapping_HasKey(context, key)); -} - -static PyObject * -sequence_set_slice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - - int res = PySequence_SetSlice(sequence, i1, i2, obj); - if (res == -1) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -sequence_del_slice(PyObject* self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - - int res = PySequence_DelSlice(sequence, i1, i2); - if (res == -1) { - return NULL; - } - Py_RETURN_NONE; -} - static PyObject * test_pythread_tss_key_state(PyObject *self, PyObject *args) { @@ -2247,72 +2163,6 @@ negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) #endif -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - return PySequence_GetItem(seq, i); -} - - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - if (PySequence_SetItem(seq, i, val)) { - return NULL; - } - Py_RETURN_NONE; -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - if (PySequence_DelItem(seq, i)) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -hasattr_string(PyObject *self, PyObject* args) -{ - PyObject* obj; - PyObject* attr_name; - - if (!PyArg_UnpackTuple(args, "hasattr_string", 2, 2, &obj, &attr_name)) { - return NULL; - } - - if (!PyUnicode_Check(attr_name)) { - PyErr_SetString(PyExc_TypeError, "attribute name must a be string"); - return PyErr_Occurred(); - } - - const char *name_str = PyUnicode_AsUTF8(attr_name); - if (PyObject_HasAttrString(obj, name_str)) { - Py_RETURN_TRUE; - } - else { - Py_RETURN_FALSE; - } -} - - /* Functions for testing C calling conventions (METH_*) are named meth_*, * e.g. "meth_varargs" for METH_VARARGS. * @@ -3736,23 +3586,12 @@ static PyMethodDef TestMethods[] = { #ifdef W_STOPCODE {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif - {"get_mapping_keys", get_mapping_keys, METH_O}, - {"get_mapping_values", get_mapping_values, METH_O}, - {"get_mapping_items", get_mapping_items, METH_O}, - {"test_mapping_has_key_string", test_mapping_has_key_string, METH_NOARGS}, - {"mapping_has_key", mapping_has_key, METH_VARARGS}, - {"sequence_set_slice", sequence_set_slice, METH_VARARGS}, - {"sequence_del_slice", sequence_del_slice, METH_VARARGS}, {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, {"hamt", new_hamt, METH_NOARGS}, {"bad_get", _PyCFunction_CAST(bad_get), METH_FASTCALL}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"hasattr_string", hasattr_string, METH_VARARGS}, {"meth_varargs", meth_varargs, METH_VARARGS}, {"meth_varargs_keywords", _PyCFunction_CAST(meth_varargs_keywords), METH_VARARGS|METH_KEYWORDS}, {"meth_o", meth_o, METH_O}, @@ -4394,6 +4233,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Heaptype(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Abstract(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Unicode(m) < 0) { return NULL; } @@ -4418,6 +4260,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Float(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Dict(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Structmember(m) < 0) { return NULL; } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index de17d74c52e56f..8c0fd0cf052b0e 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -99,7 +99,9 @@ + + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 637f7178d39d0e..87d33ebe28e475 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -27,9 +27,15 @@ Source Files + + Source Files + Source Files + + Source Files + Source Files From 430632d6f710c99879c5d1736f3b40ea09b11c4d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 7 Aug 2023 13:14:56 -0600 Subject: [PATCH 061/598] gh-107630: Initialize Each Interpreter's refchain Properly (gh-107733) This finishes fixing the crashes in Py_TRACE_REFS builds. We missed this part in gh-107567. --- Include/internal/pycore_object.h | 1 + Objects/object.c | 22 +++++++++++++++++++++- Python/pylifecycle.c | 2 ++ Python/pystate.c | 1 + 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 76b3cd69cbf5a7..7cdf64bcdacb9f 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -173,6 +173,7 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); +extern void _PyObject_InitState(PyInterpreterState *interp); /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and diff --git a/Objects/object.c b/Objects/object.c index c4413a00ede80c..d1154eb344fab1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -162,6 +162,14 @@ _PyDebug_PrintTotalRefs(void) { #define REFCHAIN(interp) &interp->object_state.refchain +static inline void +init_refchain(PyInterpreterState *interp) +{ + PyObject *refchain = REFCHAIN(interp); + refchain->_ob_prev = refchain; + refchain->_ob_next = refchain; +} + /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If * force is false amd _ob_prev or _ob_next are non-NULL, do nothing. @@ -2019,6 +2027,18 @@ PyObject _Py_NotImplementedStruct = { &_PyNotImplemented_Type }; + +void +_PyObject_InitState(PyInterpreterState *interp) +{ +#ifdef Py_TRACE_REFS + if (!_Py_IsMainInterpreter(interp)) { + init_refchain(interp); + } +#endif +} + + extern PyTypeObject _Py_GenericAliasIterType; extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; @@ -2326,7 +2346,7 @@ _Py_GetObjects(PyObject *self, PyObject *args) #undef REFCHAIN -#endif +#endif /* Py_TRACE_REFS */ /* Hack to force loading of abstract.o */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7a17f92b550c05..0de3abf9407899 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2075,6 +2075,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } has_gil = 1; + /* No objects have been created yet. */ + status = pycore_interp_init(tstate); if (_PyStatus_EXCEPTION(status)) { goto error; diff --git a/Python/pystate.c b/Python/pystate.c index a1864cc3249e2f..3a05cb0fa7988d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -674,6 +674,7 @@ init_interpreter(PyInterpreterState *interp, _obmalloc_pools_INIT(interp->obmalloc.pools); memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } + _PyObject_InitState(interp); _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); From bea5f93196d213d6fbf4ba8984caf4c3cd1da882 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 22:29:01 +0300 Subject: [PATCH 062/598] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) --- Lib/test/test_capi/test_misc.py | 43 +++++++++++++++++++++++++++++++++ Modules/_testcapimodule.c | 41 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4ab16924478043..844dd2a1020967 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -51,6 +51,8 @@ import _testinternalcapi +NULL = None + def decode_stderr(err): return err.decode('utf-8', 'replace').replace('\r', '') @@ -2576,5 +2578,46 @@ def testfunc(it): with self.assertRaises(StopIteration): next(it) + def test_sys_getobject(self): + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + def test_sys_setobject(self): + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 23dd2cc9ad40f5..35599f8baa204d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -44,6 +44,16 @@ // Include definitions from there. #include "_testcapi/parts.h" +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + // Forward declarations static struct PyModuleDef _testcapimodule; static PyObject *TestError; /* set to exception object in init */ @@ -3505,6 +3515,35 @@ test_dict_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +static PyObject * +sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + PyObject *result = PySys_GetObject(name); + if (result == NULL) { + result = PyExc_AttributeError; + } + return Py_NewRef(result); +} + +static PyObject * +sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *value; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + return NULL; + } + NULLABLE(value); + RETURN_INT(PySys_SetObject(name, value)); +} + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3640,6 +3679,8 @@ static PyMethodDef TestMethods[] = { {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, {"test_dict_capi", test_dict_capi, METH_NOARGS}, + {"sys_getobject", sys_getobject, METH_O}, + {"sys_setobject", sys_setobject, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; From 0e6e32fb84b2f7cb668e0b9927637587081e38cd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 23:34:53 +0300 Subject: [PATCH 063/598] gh-86457: Fix signature for code.replace() (GH-23199) Also add support of @text_signature in Argument Clinic. --- ...3-08-07-16-30-48.gh-issue-95065.-im4R5.rst | 2 + Objects/clinic/codeobject.c.h | 34 +-- Objects/codeobject.c | 58 ++-- Tools/clinic/clinic.py | 257 +++++++++--------- 4 files changed, 178 insertions(+), 173 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst new file mode 100644 index 00000000000000..41cfd72cb36760 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -0,0 +1,2 @@ +Argument Clinic now supports overriding automatically generated signature by +using directive `@text_signature`. diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 1f2ab56775a1ee..511a8e4aaffea4 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -154,12 +154,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(code_replace__doc__, -"replace($self, /, *, co_argcount=-1, co_posonlyargcount=-1,\n" -" co_kwonlyargcount=-1, co_nlocals=-1, co_stacksize=-1,\n" -" co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n" -" co_names=None, co_varnames=None, co_freevars=None,\n" -" co_cellvars=None, co_filename=None, co_name=None,\n" -" co_qualname=None, co_linetable=None, co_exceptiontable=None)\n" +"replace($self, /, **changes)\n" "--\n" "\n" "Return a copy of the code object with new values for the specified fields."); @@ -171,13 +166,12 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable); + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable); static PyObject * code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -217,7 +211,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = NULL; + PyObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; PyObject *co_varnames = NULL; @@ -226,8 +220,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; - PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable; - PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable; + PyObject *co_linetable = self->co_linetable; + PyObject *co_exceptiontable = self->co_exceptiontable; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); if (!args) { @@ -304,7 +298,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_code'", "bytes", args[7]); goto exit; } - co_code = (PyBytesObject *)args[7]; + co_code = args[7]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -394,7 +388,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_linetable'", "bytes", args[16]); goto exit; } - co_linetable = (PyBytesObject *)args[16]; + co_linetable = args[16]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -403,7 +397,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[17]); goto exit; } - co_exceptiontable = (PyBytesObject *)args[17]; + co_exceptiontable = args[17]; skip_optional_kwonly: return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); @@ -470,4 +464,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=4ca4c0c403dbfa71 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=16c95266bbc4bc03 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 81fa33962e0101..6987a2382d81c2 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1968,27 +1968,28 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) } /*[clinic input] +@text_signature "($self, /, **changes)" code.replace * - co_argcount: int(c_default="self->co_argcount") = -1 - co_posonlyargcount: int(c_default="self->co_posonlyargcount") = -1 - co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = -1 - co_nlocals: int(c_default="self->co_nlocals") = -1 - co_stacksize: int(c_default="self->co_stacksize") = -1 - co_flags: int(c_default="self->co_flags") = -1 - co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="NULL") = None - co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None - co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_filename: unicode(c_default="self->co_filename") = None - co_name: unicode(c_default="self->co_name") = None - co_qualname: unicode(c_default="self->co_qualname") = None - co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None - co_exceptiontable: PyBytesObject(c_default="(PyBytesObject *)self->co_exceptiontable") = None + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged + co_nlocals: int(c_default="self->co_nlocals") = unchanged + co_stacksize: int(c_default="self->co_stacksize") = unchanged + co_flags: int(c_default="self->co_flags") = unchanged + co_firstlineno: int(c_default="self->co_firstlineno") = unchanged + co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged + co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged + co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_filename: unicode(c_default="self->co_filename") = unchanged + co_name: unicode(c_default="self->co_name") = unchanged + co_qualname: unicode(c_default="self->co_qualname") = unchanged + co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged + co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged Return a copy of the code object with new values for the specified fields. [clinic start generated code]*/ @@ -1997,14 +1998,13 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=b6cd9988391d5711 input=f6f68e03571f8d7c]*/ + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable) +/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -2029,7 +2029,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, if (code == NULL) { return NULL; } - co_code = (PyBytesObject *)code; + co_code = code; } if (PySys_Audit("code.__new__", "OOOiiiiii", @@ -2068,10 +2068,10 @@ code_replace_impl(PyCodeObject *self, int co_argcount, co = PyCode_NewWithPosOnlyArgs( co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, - co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names, + co_stacksize, co_flags, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_firstlineno, - (PyObject*)co_linetable, (PyObject*)co_exceptiontable); + co_linetable, co_exceptiontable); error: Py_XDECREF(code); diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3a8431cb67efea..57ad4e41ba24a2 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4595,6 +4595,7 @@ def reset(self) -> None: self.indent = IndentStack() self.kind = CALLABLE self.coexist = False + self.forced_text_signature: str | None = None self.parameter_continuation = '' self.preserve_output = False @@ -4735,6 +4736,11 @@ def at_coexist(self) -> None: fail("Called @coexist twice!") self.coexist = True + def at_text_signature(self, text_signature: str) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + def parse(self, block: Block) -> None: self.reset() self.block = block @@ -5515,142 +5521,145 @@ def format_docstring(self) -> str: add(f.cls.name) else: add(f.name) - add('(') - - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - assert parameters[0].right_bracket_count == 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired: int) -> str: - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - - need_slash = False - added_slash = False - need_a_trailing_slash = False - - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): - if not p.converter.show_in_signature: - continue - if p.is_positional_only(): - need_a_trailing_slash = True - break + if self.forced_text_signature: + add(self.forced_text_signature) + else: + add('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + assert parameters[0].right_bracket_count == 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired: int) -> str: + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break - added_star = False + added_star = False - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(text)) - indent = " " * line_length - def add_parameter(text: str) -> None: - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - add('\n') - add(indent) - line_length = len(indent) + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(text)) + indent = " " * line_length + def add_parameter(text: str) -> None: + nonlocal line_length + nonlocal first_parameter + if first_parameter: s = text - line_length += len(s) - add(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + add('\n') + add(indent) + line_length = len(indent) + s = text + line_length += len(s) + add(s) + + for p in parameters: + if not p.converter.show_in_signature: + continue + assert p.name - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_add('$') + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_add, p_output = text_accumulator() + p_add(fix_right_bracket_count(p.right_bracket_count)) + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_add('$') - if p.is_vararg(): - p_add("*") + if p.is_vararg(): + p_add("*") - name = p.converter.signature_name or p.name - p_add(name) + name = p.converter.signature_name or p.name + p_add(name) - if not p.is_vararg() and p.converter.is_optional(): - p_add('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_add(value) + if not p.is_vararg() and p.converter.is_optional(): + p_add('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_add(value) - if (p != last_p) or need_a_trailing_slash: - p_add(',') + if (p != last_p) or need_a_trailing_slash: + p_add(',') - add_parameter(p_output()) + add_parameter(p_output()) - add(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - add(')') + add(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + add(')') # PEP 8 says: # From 0191af97a6bf3f720cd0ae69a0bdb14c97351679 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 8 Aug 2023 00:04:11 +0300 Subject: [PATCH 064/598] gh-107735: Move just added C API tests to better place (GH-107743) --- Lib/test/test_capi/test_misc.py | 80 ++++++++++++++++----------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 844dd2a1020967..001d37de8e0eb3 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -992,6 +992,46 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) + def test_sys_getobject(self): + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + def test_sys_setobject(self): + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): @@ -2578,46 +2618,6 @@ def testfunc(it): with self.assertRaises(StopIteration): next(it) - def test_sys_getobject(self): - getobject = _testcapi.sys_getobject - - self.assertIs(getobject(b'stdout'), sys.stdout) - with support.swap_attr(sys, '\U0001f40d', 42): - self.assertEqual(getobject('\U0001f40d'.encode()), 42) - - self.assertIs(getobject(b'nonexisting'), AttributeError) - self.assertIs(getobject(b'\xff'), AttributeError) - # CRASHES getobject(NULL) - - def test_sys_setobject(self): - setobject = _testcapi.sys_setobject - - value = ['value'] - value2 = ['value2'] - try: - self.assertEqual(setobject(b'newattr', value), 0) - self.assertIs(sys.newattr, value) - self.assertEqual(setobject(b'newattr', value2), 0) - self.assertIs(sys.newattr, value2) - self.assertEqual(setobject(b'newattr', NULL), 0) - self.assertFalse(hasattr(sys, 'newattr')) - self.assertEqual(setobject(b'newattr', NULL), 0) - finally: - with contextlib.suppress(AttributeError): - del sys.newattr - try: - self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) - self.assertIs(getattr(sys, '\U0001f40d'), value) - self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) - self.assertFalse(hasattr(sys, '\U0001f40d')) - finally: - with contextlib.suppress(AttributeError): - delattr(sys, '\U0001f40d') - - with self.assertRaises(UnicodeDecodeError): - setobject(b'\xff', value) - # CRASHES setobject(NULL, value) - if __name__ == "__main__": unittest.main() From 5dc825d504ad08d64c9d1ce578f9deebbe012604 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 7 Aug 2023 15:40:59 -0600 Subject: [PATCH 065/598] gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107552) PEP 683 (immortal objects) revealed some ways in which the Python documentation has been unnecessarily coupled to the implementation details of reference counts. In the end users should focus on reference ownership, including taking references and releasing them, rather than on how many reference counts an object has. This change updates the documentation to reflect that perspective. It also updates the docs relative to immortal objects in a handful of places. --- Doc/c-api/allocation.rst | 13 ++++--- Doc/c-api/arg.rst | 17 +++++--- Doc/c-api/buffer.rst | 7 +++- Doc/c-api/bytes.rst | 4 +- Doc/c-api/exceptions.rst | 3 +- Doc/c-api/init_config.rst | 2 +- Doc/c-api/intro.rst | 59 +++++++++++++++------------- Doc/c-api/module.rst | 2 +- Doc/c-api/object.rst | 13 ++++--- Doc/c-api/refcounting.rst | 81 ++++++++++++++++++++++++++++----------- Doc/c-api/sys.rst | 5 ++- Doc/c-api/typeobj.rst | 10 +++-- Doc/c-api/unicode.rst | 12 +++--- Doc/glossary.rst | 15 +++++--- Doc/library/sys.rst | 9 +++++ 15 files changed, 162 insertions(+), 90 deletions(-) diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 44747e29643661..b3609c233156b6 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -29,12 +29,13 @@ Allocating Objects on the Heap .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. + Allocate a new Python object using the C structure type *TYPE* + and the Python type object *typeobj* (``PyTypeObject*``). + Fields not defined by the Python object header are not initialized. + The caller will own the only reference to the object + (i.e. its reference count will be one). + The size of the memory allocation is determined from the + :c:member:`~PyTypeObject.tp_basicsize` field of the type object. .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index ae321ff918d13c..c43dd0f4303cd4 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -293,8 +293,10 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not ``NULL``. + program thus receives the actual object that was passed. A new + :term:`strong reference` to the object is not created + (i.e. its reference count is not increased). + The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but @@ -378,7 +380,8 @@ inside nested parentheses. They are: mutually exclude each other. Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! +*borrowed* references; do not release them +(i.e. do not decrement their reference count)! Additional arguments passed to these functions must be addresses of variables whose type is determined by the format string; these are used to store values @@ -621,8 +624,10 @@ Building values Convert a C :c:type:`Py_complex` structure to a Python complex number. ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed + Pass a Python object untouched but create a new + :term:`strong reference` to it + (i.e. its reference count is incremented by one). + If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is @@ -632,7 +637,7 @@ Building values Same as ``O``. ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. + Same as ``O``, except it doesn't create a new :term:`strong reference`. Useful when the object is created by a call to an object constructor in the argument list. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 02b53ec149c733..8ca1c190dab9a9 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to ``NULL`` by + the consumer and automatically released + (i.e. reference count decremented) + and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. @@ -454,7 +456,8 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) - Release the buffer *view* and decrement the reference count for + Release the buffer *view* and release the :term:`strong reference` + (i.e. decrement the reference count) to the view's supporting object, ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 4e3ffc7e23e3f8..61a68f52773882 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -184,8 +184,8 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. + appended to *bytes*. This version releases the :term:`strong reference` + to *newpart* (i.e. decrements its reference count). .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a2126ffc559aba..f1d6c995188abb 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -110,7 +110,8 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + e.g. :c:data:`PyExc_RuntimeError`. You need not create a new + :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). The second argument is an error message; it is decoded from ``'utf-8'``. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index ac7b357e08eeba..3ad2d435665f89 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1128,7 +1128,7 @@ PyConfig .. c:member:: int show_ref_count - Show total reference count at exit? + Show total reference count at exit (excluding immortal objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 1739eabd2a3039..5cf9914be8c8bf 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -287,52 +287,58 @@ true if (and only if) the object pointed to by *a* is a Python list. Reference Counts ---------------- -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") +The reference count is important because today's computers have a finite +(and often severely limited) memory size; it counts how many different +places there are that have a :term:`strong reference` to an object. +Such a place could be another object, or a global (or static) C variable, +or a local variable in some C function. +When the last :term:`strong reference` to an object is released +(i.e. its reference count becomes zero), the object is deallocated. +If it contains references to other objects, those references are released. +Those other objects may be deallocated in turn, if there are no more +references to them, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution +is "don't do that.") .. index:: single: Py_INCREF() single: Py_DECREF() -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +Reference counts are always manipulated explicitly. The normal way is +to use the macro :c:func:`Py_INCREF` to take a new reference to an +object (i.e. increment its reference count by one), +and :c:func:`Py_DECREF` to release that reference (i.e. decrement the +reference count by one). The :c:func:`Py_DECREF` macro is considerably more complex than the incref one, since it must check whether the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of releasing references +for other objects contained in the object if this is a compound object type, such as a list, as well as performing any additional finalization that's needed. There's no chance that the reference count can overflow; at least as many bits are used to hold the reference count as there are distinct memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). Thus, the reference count increment is a simple operation. -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's +It is not necessary to hold a :term:`strong reference` (i.e. increment +the reference count) for every local variable that contains a pointer +to an object. In theory, the object's reference count goes up by one when the variable is made to point to it and it goes down by one when the variable goes out of scope. However, these two cancel each other out, so at the end the reference count hasn't changed. The only real reason to use the reference count is to prevent the object from being deallocated as long as our variable is pointing to it. If we know that there is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. +our variable, there is no need to take a new :term:`strong reference` +(i.e. increment the reference count) temporarily. An important situation where this arises is in objects that are passed as arguments to C functions in an extension module that are called from Python; the call mechanism guarantees to hold a reference to every argument for the duration of the call. However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count +for a while without taking a new reference. Some other operation might +conceivably remove the object from the list, releasing that reference, and possibly deallocating it. The real danger is that innocent-looking operations may invoke arbitrary Python code which could do this; there is a code path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so @@ -340,7 +346,8 @@ almost any operation is potentially dangerous. A safe approach is to always use the generic operations (functions whose name begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. +These operations always create a new :term:`strong reference` +(i.e. increment the reference count) of the object they return. This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when they are done with the result; this soon becomes second nature. @@ -356,7 +363,7 @@ to objects (objects are not owned: they are always shared). "Owning a reference" means being responsible for calling Py_DECREF on it when the reference is no longer needed. Ownership can also be transferred, meaning that the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, @@ -414,9 +421,9 @@ For example, the above two blocks of code could be replaced by the following It is much more common to use :c:func:`PyObject_SetItem` and friends with items whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function +the function you are writing. In that case, their behaviour regarding references +is much saner, since you don't have to take a new reference just so you +can give that reference away ("have it be stolen"). For example, this function sets all items of a list (actually, any mutable sequence) to a given item:: int diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index e37505f6450a30..187f8419d4ee4f 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -546,7 +546,7 @@ state: .. note:: Unlike other functions that steal references, ``PyModule_AddObject()`` - only decrements the reference count of *value* **on success**. + only releases the reference to *value* **on success**. This means that its return value must be checked, and calling code must :c:func:`Py_XDECREF` *value* manually on error. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 1b7e05e7c53eda..6d9fc7f50604da 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -15,8 +15,8 @@ Object Protocol .. c:macro:: Py_RETURN_NOTIMPLEMENTED Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). + function (that is, create a new :term:`strong reference` + to NotImplemented and return it). .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -357,11 +357,12 @@ Object Protocol When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this + is equivalent to the Python expression ``type(o)``. + This function creates a new :term:`strong reference` to the return value. + There's really no reason to use this function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference - count is needed. + pointer of type :c:expr:`PyTypeObject*`, except when a new + :term:`strong reference` is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 640c5c610899f8..118af7a1a8cf90 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -15,6 +15,12 @@ of Python objects. Get the reference count of the Python object *o*. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are "immortal" and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. .. versionchanged:: 3.11 @@ -28,36 +34,53 @@ of Python objects. Set the object *o* reference counter to *refcnt*. + Note that this function has no effect on + `immortal `_ + objects. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. + Indicate taking a new :term:`strong reference` to object *o*, + indicating it is in use and should not be destroyed. This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. + When done using the object, release is by calling :c:func:`Py_DECREF`. + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect. + Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, + in which case this has no effect. See also :c:func:`Py_XNewRef`. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Create a new :term:`strong reference` to an object: increment the reference - count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: + call :c:func:`Py_INCREF` on *o* and return the object *o*. When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to decrement the object reference count. + should be called on it to release the reference. The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be ``NULL``. @@ -87,9 +110,12 @@ of Python objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. + Release a :term:`strong reference` to object *o*, indicating the + reference is no longer used. - If the reference count reaches zero, the object's type's deallocation + Once the last :term:`strong reference` is released + (i.e. the object's reference count reaches 0), + the object's type's deallocation function (which must not be ``NULL``) is invoked. This function is usually used to delete a :term:`strong reference` before @@ -98,6 +124,10 @@ of Python objects. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. @@ -109,25 +139,29 @@ of Python objects. reference to the deleted object in a temporary variable, update the list data structure, and then call :c:func:`Py_DECREF` for the temporary variable. + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. + Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, + in which case this has no effect. + The same warning from :c:func:`Py_DECREF` applies here as well. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in + Release a :term:`strong reference` for object *o*. + The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before decrementing its reference count. + before releasing the reference. - It is a good idea to use this macro whenever decrementing the reference - count of an object that might be traversed during garbage collection. + It is a good idea to use this macro whenever releasing a reference + to an object that might be traversed during garbage collection. .. versionchanged:: 3.12 The macro argument is now only evaluated once. If the argument has side @@ -136,20 +170,22 @@ of Python objects. .. c:function:: void Py_IncRef(PyObject *o) - Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + Indicate taking a new :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XINCREF`. It can be used for runtime dynamic embedding of Python. .. c:function:: void Py_DecRef(PyObject *o) - Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + Release a :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. .. c:macro:: Py_SETREF(dst, src) - Macro safely decrementing the `dst` reference count and setting `dst` to - `src`. + Macro safely releasing a :term:`strong reference` to object *dst* + and setting *dst* to *src*. As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly:: @@ -160,9 +196,10 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set `dst` to `src` _before_ decrementing reference count of - *dst* old value, so that any code triggered as a side-effect of `dst` - getting torn down no longer believes `dst` points to a valid object. + That arranges to set *dst* to *src* _before_ releasing the reference + to the old value of *dst*, so that any code triggered as a side-effect + of *dst* getting torn down no longer believes *dst* points + to a valid object. .. versionadded:: 3.6 diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 5457d17159f8d3..aed625c5f6cdae 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -8,8 +8,9 @@ Operating System Utilities .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, + :class:`str` or :class:`bytes` object, then a new + :term:`strong reference` is returned. + If the object implements the :class:`os.PathLike` interface, then :meth:`~os.PathLike.__fspath__` is returned as long as it is a :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised and ``NULL`` is returned. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 26e6133aebaa8f..221a05b1922404 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -690,7 +690,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) } Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should decrement the reference count for its type object after + deallocator should release the owned reference to its type object + (via :c:func:`Py_DECREF`) after calling the type deallocator. In order to avoid dangling pointers, the recommended way to achieve this is: @@ -1461,9 +1462,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) } The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until + delicate: the reference to the contained object must not be released + (via :c:func:`Py_DECREF`) until after the pointer to the contained object is set to ``NULL``. This is because - decrementing the reference count may cause the contained object to become trash, + releasing the reference may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, @@ -1541,7 +1543,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) they may be C ints or floats). The third argument specifies the requested operation, as for :c:func:`PyObject_RichCompare`. - The return value's reference count is properly incremented. + The returned value is a new :term:`strong reference`. On error, sets an exception and returns ``NULL`` from the function. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index e875cf6ded655e..cf965fa1676524 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -564,7 +564,7 @@ APIs: Copy an instance of a Unicode subtype to a new true Unicode object if necessary. If *obj* is already a true Unicode object (not a subtype), - return the reference with incremented refcount. + return a new :term:`strong reference` to the object. Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. @@ -1438,11 +1438,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Intern the argument *\*string* in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call + it (releasing the reference to the old string object and creating a new + :term:`strong reference` to the interned string object), otherwise it leaves + *\*string* alone and interns it (creating a new :term:`strong reference`). + (Clarification: even though there is a lot of talk about references, think + of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a4650a6c3efa22..a4cd05b0cf019d 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -159,8 +159,9 @@ Glossary :class:`str` objects. borrowed reference - In Python's C API, a borrowed reference is a reference to an object. - It does not modify the object reference count. It becomes a dangling + In Python's C API, a borrowed reference is a reference to an object, + where the code using the object does not own the reference. + It becomes a dangling pointer if the object is destroyed. For example, a garbage collection can remove the last :term:`strong reference` to the object and so destroy it. @@ -1054,7 +1055,9 @@ Glossary reference count The number of references to an object. When the reference count of an - object drops to zero, it is deallocated. Reference counting is + object drops to zero, it is deallocated. Some objects are + "immortal" and have reference counts that are never modified, and + therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the :func:`sys.getrefcount` function to return the @@ -1137,8 +1140,10 @@ Glossary strong reference In Python's C API, a strong reference is a reference to an object - which increments the object's reference count when it is created and - decrements the object's reference count when it is deleted. + which is owned by the code holding the reference. The strong + reference is taken by calling :c:func:`Py_INCREF` when the + reference is created and released with :c:func:`Py_DECREF` + when the reference is deleted. The :c:func:`Py_NewRef` function can be used to create a strong reference to an object. Usually, the :c:func:`Py_DECREF` function must be called on diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a61737383e3939..33391d11ab392d 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -770,6 +770,15 @@ always available. higher than you might expect, because it includes the (temporary) reference as an argument to :func:`getrefcount`. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are "immortal" and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + + .. versionchanged:: 3.12 + Immortal objects have very large refcounts that do not match + the actual number of references to the object. .. function:: getrecursionlimit() From 707018cc75558d6695e9e199a3ed0c8a4ff7cbcc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 7 Aug 2023 17:10:57 -0600 Subject: [PATCH 066/598] gh-107630: Fix Remaining Subinterpreters Crashes on Py_TRACE_REFS Builds (gh-107750) This is a follow-up to gh-107567 and gh-107733. We skip test_basic_multiple_interpreters_deleted_no_reset on tracerefs builds. The test breaks interpreter isolation a little, which doesn't work well with Py_TRACE_REFS builds, so I feel fine about skipping the test. --- Lib/test/test_import/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 163ed824ff1fb0..051711bfd1fe24 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2555,6 +2555,12 @@ def test_basic_multiple_interpreters_main_no_reset(self): def test_basic_multiple_interpreters_deleted_no_reset(self): # without resetting; already loaded in a deleted interpreter + if hasattr(sys, 'getobjects'): + # It's a Py_TRACE_REFS build. + # This test breaks interpreter isolation a little, + # which causes problems on Py_TRACE_REF builds. + raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') + # At this point: # * alive in 0 interpreters # * module def may or may not be loaded already From 2df58dcd500dbedc61d0630374f9e94c522fe523 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 7 Aug 2023 21:32:42 -0700 Subject: [PATCH 067/598] gh-106812: Small stack effect fixes (#107759) - Generalize the syntax for the type of a stack effect to allow a trailing `*`, so we can declare something as e.g. `PyCodeObject *`. - When generating assignments for stack effects, the type of the value on the stack should be the default (i.e., `PyObject *`) even when the variable copied to/from it has a different type, so that an appropriate cast is generated However, not when the variable is an array -- then the type is taken from the variable (as it is always `PyObject **`). --- .../cases_generator/interpreter_definition.md | 2 +- Tools/cases_generator/parsing.py | 4 ++- Tools/cases_generator/stacking.py | 29 +++++++------------ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index f141848631d04a..5c4238756748a7 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -108,7 +108,7 @@ and a piece of C code describing its semantics:: NAME [":" type] [ "if" "(" C-expression ")" ] type: - NAME + NAME ["*"] stream: NAME "/" size diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 5610ac661a8945..cdd20d7a0b3f59 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -252,12 +252,14 @@ def cache_effect(self) -> CacheEffect | None: @contextual def stack_effect(self) -> StackEffect | None: - # IDENTIFIER [':' IDENTIFIER] ['if' '(' expression ')'] + # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): type_text = "" if self.expect(lx.COLON): type_text = self.require(lx.IDENTIFIER).text.strip() + if self.expect(lx.TIMES): + type_text += " *" cond_text = "" if self.expect(lx.IF): self.require(lx.LPAREN) diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 23eca3037f896d..d457ce01a8f438 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -120,6 +120,14 @@ def as_variable(self, lax: bool = False) -> str: ), f"Push or pop above current stack level: {res}" return res + def as_stack_effect(self, lax: bool = False) -> StackEffect: + return StackEffect( + self.as_variable(lax=lax), + self.effect.type if self.effect.size else "", + self.effect.cond, + self.effect.size, + ) + @dataclasses.dataclass class CopyEffect: @@ -356,24 +364,14 @@ def write_components( for peek in mgr.peeks: out.assign( peek.effect, - StackEffect( - peek.as_variable(), - peek.effect.type, - peek.effect.cond, - peek.effect.size, - ), + peek.as_stack_effect(), ) # Initialize array outputs for poke in mgr.pokes: if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: out.assign( poke.effect, - StackEffect( - poke.as_variable(lax=True), - poke.effect.type, - poke.effect.cond, - poke.effect.size, - ), + poke.as_stack_effect(lax=True), ) if len(parts) == 1: @@ -390,11 +388,6 @@ def write_components( for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: out.assign( - StackEffect( - poke.as_variable(), - poke.effect.type, - poke.effect.cond, - poke.effect.size, - ), + poke.as_stack_effect(), poke.effect, ) From 328d925244511b2134d5ac926e307e4486ff4500 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 7 Aug 2023 21:36:25 -0700 Subject: [PATCH 068/598] gh-107758: Improvements to lltrace feature (#107757) - The `dump_stack()` method could call a `__repr__` method implemented in Python, causing (infinite) recursion. I rewrote it to only print out the values for some fundamental types (`int`, `str`, etc.); for everything else it just prints ``. - The lltrace-like feature for uops wrote to `stderr`, while the one in `ceval.c` writes to `stdout`; I changed the uops to write to stdout as well. --- ...-08-08-02-46-46.gh-issue-107758.R5kyBI.rst | 1 + Python/ceval.c | 34 +++++++++++++++---- Python/executor.c | 2 +- Python/optimizer.c | 2 +- 4 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst new file mode 100644 index 00000000000000..192f1df26e613e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst @@ -0,0 +1 @@ +Make the ``dump_stack()`` routine used by the ``lltrace`` feature (low-level interpreter debugging) robust against recursion by ensuring that it never calls a ``__repr__`` method implemented in Python. Also make the similar output for Tier-2 uops appear on ``stdout`` (instead of ``stderr``), to match the ``lltrace`` code in ceval.c. diff --git a/Python/ceval.c b/Python/ceval.c index 30f722e45e476b..1dc4fd80224bbd 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -109,11 +109,24 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) if (ptr != stack_base) { printf(", "); } - if (PyObject_Print(*ptr, stdout, 0) != 0) { + if (*ptr == NULL) { + printf(""); + continue; + } + if ( + *ptr == Py_None + || PyBool_Check(*ptr) + || PyLong_CheckExact(*ptr) + || PyFloat_CheckExact(*ptr) + || PyUnicode_CheckExact(*ptr) + ) { + if (PyObject_Print(*ptr, stdout, 0) == 0) { + continue; + } PyErr_Clear(); - printf("<%s object at %p>", - Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); } printf("]\n"); fflush(stdout); @@ -128,9 +141,6 @@ lltrace_instruction(_PyInterpreterFrame *frame, if (frame->owner == FRAME_OWNED_BY_CSTACK) { return; } - /* This dump_stack() operation is risky, since the repr() of some - objects enters the interpreter recursively. It is also slow. - So you might want to comment it out. */ dump_stack(frame, stack_pointer); int oparg = next_instr->op.arg; int opcode = next_instr->op.code; @@ -729,6 +739,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto exit_unwind; } lltrace = r; + if (!lltrace) { + // When tracing executed uops, also trace bytecode + char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = (*uop_debug - '0') >= 4; // TODO: Parse an int and all that + } + } } if (lltrace) { lltrace_resume_frame(frame); @@ -896,6 +913,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto exception_unwind; } /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace) { + lltrace_resume_frame(frame); + } +#endif DISPATCH(); } } diff --git a/Python/executor.c b/Python/executor.c index 57525df202d861..4a18618c0c6c0c 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -41,7 +41,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject lltrace = *uop_debug - '0'; // TODO: Parse an int and all that } #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } + if (lltrace >= (level)) { printf(__VA_ARGS__); } #else #define DPRINTF(level, ...) #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index d2ed8dfcd2cff2..6c730aa14b9a47 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -391,7 +391,7 @@ translate_bytecode_to_trace( #ifdef Py_DEBUG #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } + if (lltrace >= (level)) { printf(__VA_ARGS__); } #else #define DPRINTF(level, ...) #endif From 0db043dd5a34f7b5968476011e36396737d09030 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 8 Aug 2023 08:41:09 +0200 Subject: [PATCH 069/598] gh-95065: Make Argument Clinic append deprecation warnings to docstrings (#107745) --- Lib/test/clinic.test.c | 75 ++++++++++++++++++++++++++++++++---------- Tools/clinic/clinic.py | 6 ++++ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index c5c37b7e9e2168..386d1dac3f115e 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5477,7 +5477,11 @@ test_deprecate_positional_pos1_len1_optional PyDoc_STRVAR(test_deprecate_positional_pos1_len1_optional__doc__, "test_deprecate_positional_pos1_len1_optional($module, /, a, b=None)\n" "--\n" -"\n"); +"\n" +"Note: Passing 2 positional arguments to\n" +"test_deprecate_positional_pos1_len1_optional() is deprecated.\n" +"Parameter \'b\' will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_OPTIONAL_METHODDEF \ {"test_deprecate_positional_pos1_len1_optional", _PyCFunction_CAST(test_deprecate_positional_pos1_len1_optional), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1_optional__doc__}, @@ -5567,7 +5571,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * static PyObject * test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=09a6edec1ddcd469 input=89099f3dacd757da]*/ +/*[clinic end generated code: output=6c0fd9d94fa1e765 input=89099f3dacd757da]*/ /*[clinic input] @@ -5580,7 +5584,11 @@ test_deprecate_positional_pos1_len1 PyDoc_STRVAR(test_deprecate_positional_pos1_len1__doc__, "test_deprecate_positional_pos1_len1($module, /, a, b)\n" "--\n" -"\n"); +"\n" +"Note: Passing 2 positional arguments to\n" +"test_deprecate_positional_pos1_len1() is deprecated. Parameter \'b\'\n" +"will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_METHODDEF \ {"test_deprecate_positional_pos1_len1", _PyCFunction_CAST(test_deprecate_positional_pos1_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1__doc__}, @@ -5661,7 +5669,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=52a2618293df747d input=1702bbab1e9b3b99]*/ +/*[clinic end generated code: output=22821a0fa9945d0c input=1702bbab1e9b3b99]*/ /*[clinic input] @@ -5677,7 +5685,12 @@ test_deprecate_positional_pos1_len2_with_kwd PyDoc_STRVAR(test_deprecate_positional_pos1_len2_with_kwd__doc__, "test_deprecate_positional_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" "--\n" -"\n"); +"\n" +"Note: Passing more than 1 positional argument to\n" +"test_deprecate_positional_pos1_len2_with_kwd() is deprecated.\n" +"Parameters \'b\' and \'c\' will become keyword-only parameters in Python\n" +"3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS1_LEN2_WITH_KWD_METHODDEF \ {"test_deprecate_positional_pos1_len2_with_kwd", _PyCFunction_CAST(test_deprecate_positional_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len2_with_kwd__doc__}, @@ -5768,7 +5781,7 @@ static PyObject * test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=550aabea548589b4 input=28cdb885f6c34eab]*/ +/*[clinic end generated code: output=061d554ccc6b8f51 input=28cdb885f6c34eab]*/ /*[clinic input] @@ -5780,7 +5793,11 @@ test_deprecate_positional_pos0_len1 PyDoc_STRVAR(test_deprecate_positional_pos0_len1__doc__, "test_deprecate_positional_pos0_len1($module, /, a)\n" "--\n" -"\n"); +"\n" +"Note: Passing positional arguments to\n" +"test_deprecate_positional_pos0_len1() is deprecated. Parameter \'a\'\n" +"will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS0_LEN1_METHODDEF \ {"test_deprecate_positional_pos0_len1", _PyCFunction_CAST(test_deprecate_positional_pos0_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len1__doc__}, @@ -5857,7 +5874,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=66c63ec8d6903bde input=678206db25c0652c]*/ +/*[clinic end generated code: output=3e512117a5eda970 input=678206db25c0652c]*/ /*[clinic input] @@ -5870,7 +5887,11 @@ test_deprecate_positional_pos0_len2 PyDoc_STRVAR(test_deprecate_positional_pos0_len2__doc__, "test_deprecate_positional_pos0_len2($module, /, a, b)\n" "--\n" -"\n"); +"\n" +"Note: Passing positional arguments to\n" +"test_deprecate_positional_pos0_len2() is deprecated. Parameters \'a\'\n" +"and \'b\' will become keyword-only parameters in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS0_LEN2_METHODDEF \ {"test_deprecate_positional_pos0_len2", _PyCFunction_CAST(test_deprecate_positional_pos0_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len2__doc__}, @@ -5954,7 +5975,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=6b6df40aaf751b2e input=fae0d0b1d480c939]*/ +/*[clinic end generated code: output=d41da050a5b82dd0 input=fae0d0b1d480c939]*/ /*[clinic input] @@ -5971,7 +5992,12 @@ PyDoc_STRVAR(test_deprecate_positional_pos0_len3_with_kwdonly__doc__, "test_deprecate_positional_pos0_len3_with_kwdonly($module, /, a, b, c,\n" " *, e)\n" "--\n" -"\n"); +"\n" +"Note: Passing positional arguments to\n" +"test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated.\n" +"Parameters \'a\', \'b\' and \'c\' will become keyword-only parameters in\n" +"Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS0_LEN3_WITH_KWDONLY_METHODDEF \ {"test_deprecate_positional_pos0_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos0_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len3_with_kwdonly__doc__}, @@ -6069,7 +6095,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, PyObject *b, PyObject *c, PyObject *e) -/*[clinic end generated code: output=5c936993846d01a3 input=1b0121770c0c52e0]*/ +/*[clinic end generated code: output=c5d7ddfc139ddf31 input=1b0121770c0c52e0]*/ /*[clinic input] @@ -6083,7 +6109,11 @@ test_deprecate_positional_pos2_len1 PyDoc_STRVAR(test_deprecate_positional_pos2_len1__doc__, "test_deprecate_positional_pos2_len1($module, /, a, b, c)\n" "--\n" -"\n"); +"\n" +"Note: Passing 3 positional arguments to\n" +"test_deprecate_positional_pos2_len1() is deprecated. Parameter \'c\'\n" +"will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS2_LEN1_METHODDEF \ {"test_deprecate_positional_pos2_len1", _PyCFunction_CAST(test_deprecate_positional_pos2_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len1__doc__}, @@ -6166,7 +6196,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=2641e037296e3b61 input=e1d129689e69ec7c]*/ +/*[clinic end generated code: output=d80609e6b37ffb8a input=e1d129689e69ec7c]*/ /*[clinic input] @@ -6181,7 +6211,11 @@ test_deprecate_positional_pos2_len2 PyDoc_STRVAR(test_deprecate_positional_pos2_len2__doc__, "test_deprecate_positional_pos2_len2($module, /, a, b, c, d)\n" "--\n" -"\n"); +"\n" +"Note: Passing more than 2 positional arguments to\n" +"test_deprecate_positional_pos2_len2() is deprecated. Parameters \'c\'\n" +"and \'d\' will become keyword-only parameters in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS2_LEN2_METHODDEF \ {"test_deprecate_positional_pos2_len2", _PyCFunction_CAST(test_deprecate_positional_pos2_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len2__doc__}, @@ -6271,7 +6305,7 @@ static PyObject * test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=4a9068ef8fee61f6 input=0d53533463a12792]*/ +/*[clinic end generated code: output=1c10d6197562319f input=0d53533463a12792]*/ /*[clinic input] @@ -6289,7 +6323,12 @@ PyDoc_STRVAR(test_deprecate_positional_pos2_len3_with_kwdonly__doc__, "test_deprecate_positional_pos2_len3_with_kwdonly($module, /, a, b, c,\n" " d, *, e)\n" "--\n" -"\n"); +"\n" +"Note: Passing more than 2 positional arguments to\n" +"test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated.\n" +"Parameters \'c\' and \'d\' will become keyword-only parameters in Python\n" +"3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS2_LEN3_WITH_KWDONLY_METHODDEF \ {"test_deprecate_positional_pos2_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos2_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len3_with_kwdonly__doc__}, @@ -6388,4 +6427,4 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=1154c2e3e798948c input=154fd450448d8935]*/ +/*[clinic end generated code: output=d32375ffce63d3db input=154fd450448d8935]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 57ad4e41ba24a2..82e5b804c2e3ea 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -941,6 +941,12 @@ def deprecate_positional_use( f"{func.full_name}() is deprecated. Parameters {pstr} will " f"become keyword-only parameters in Python {major}.{minor}." ) + + # Append deprecation warning to docstring. + lines = textwrap.wrap(f"Note: {depr_message}") + docstring = "\n".join(lines) + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. code = self.DEPRECATED_POSITIONAL_PROTOTYPE.format( condition=condition, From a9aeb99579f24bbce1dd553d605a5a5e2f37a3a2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 8 Aug 2023 08:42:08 +0200 Subject: [PATCH 070/598] gh-86457: Add docs for Argument Clinic @text_signature directive (#107747) Co-authored-by: Alex Waygood --- Doc/howto/clinic.rst | 34 +++++++++++++++++++ ...3-08-07-16-30-48.gh-issue-95065.-im4R5.rst | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 286623c2410145..743c7c9cb3000e 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1900,6 +1900,40 @@ blocks embedded in Python files look slightly different. They look like this: #/*[python checksum:...]*/ +.. _clinic-howto-override-signature: + +How to override the generated signature +--------------------------------------- + +You can use the ``@text_signature`` directive to override the default generated +signature in the docstring. +This can be useful for complex signatures that Argument Clinic cannot handle. +The ``@text_signature`` directive takes one argument: +the custom signature as a string. +The provided signature is copied verbatim to the generated docstring. + +Example from :source:`Objects/codeobject.c`:: + + /*[clinic input] + @text_signature "($self, /, **changes)" + code.replace + * + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + # etc ... + + Return a copy of the code object with new values for the specified fields. + [clinic start generated output]*/ + +The generated docstring ends up looking like this:: + + Doc_STRVAR(code_replace__doc__, + "replace($self, /, **changes)\n" + "--\n" + "\n" + "Return a copy of the code object with new values for the specified fields."); + + .. _clinic-howto-deprecate-positional: How to deprecate passing parameters positionally diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst index 41cfd72cb36760..df1893e9c808ef 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -1,2 +1,2 @@ Argument Clinic now supports overriding automatically generated signature by -using directive `@text_signature`. +using directive `@text_signature`. See :ref:`clinic-howto-override-signature`. From 7a250fdc16bb6f1fe0a6b0df8bb502870405b5d6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 8 Aug 2023 16:37:45 +0900 Subject: [PATCH 071/598] README: remove unmaintained sections (#107703) --- README.rst | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/README.rst b/README.rst index 4ab26565a13e03..208bf8cec444a3 100644 --- a/README.rst +++ b/README.rst @@ -211,30 +211,6 @@ primary version, you would execute ``make install`` in your 3.13 build directory and ``make altinstall`` in the others. -Issue Tracker and Mailing List ------------------------------- - -Bug reports are welcome! You can use Github to `report bugs -`_, and/or `submit pull requests -`_. - -You can also follow development discussion on the `python-dev mailing list -`_. - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -`comp.lang.python`_ or `python-ideas`_ mailing lists for initial feedback. A -Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. -All current PEPs, as well as guidelines for submitting a new PEP, are listed at -`peps.python.org `_. - -.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ -.. _comp.lang.python: https://mail.python.org/mailman/listinfo/python-list - - Release Schedule ---------------- From de72677f8a27083b2072b83b3737f891b660bb5c Mon Sep 17 00:00:00 2001 From: Tomas R Date: Tue, 8 Aug 2023 10:20:10 +0200 Subject: [PATCH 072/598] gh-107659: Add docstrings for ctypes.pointer and ctypes.POINTER (#107660) --- ...-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst | 1 + Modules/_ctypes/callproc.c | 44 ++++++++++++++++--- Modules/_ctypes/clinic/callproc.c.h | 38 ++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst create mode 100644 Modules/_ctypes/clinic/callproc.c.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst new file mode 100644 index 00000000000000..31cc6982400d5d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst @@ -0,0 +1 @@ +Add docstrings for :func:`ctypes.pointer` and :func:`ctypes.POINTER`. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 69cf8a98af6636..f9535db4f57c0e 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -54,6 +54,11 @@ */ +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -98,6 +103,7 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_global_objects.h" // _Py_ID() +#include "clinic/callproc.c.h" #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" @@ -1893,8 +1899,22 @@ unpickle(PyObject *self, PyObject *args) return NULL; } +/*[clinic input] +_ctypes.POINTER as create_pointer_type + + type as cls: object + A ctypes type. + / + +Create and return a new ctypes pointer type. + +Pointer types are cached and reused internally, +so calling this function repeatedly is cheap. +[clinic start generated code]*/ + static PyObject * -POINTER(PyObject *self, PyObject *cls) +create_pointer_type(PyObject *module, PyObject *cls) +/*[clinic end generated code: output=98c3547ab6f4f40b input=3b81cff5ff9b9d5b]*/ { PyObject *result; PyTypeObject *typ; @@ -1944,8 +1964,22 @@ POINTER(PyObject *self, PyObject *cls) return result; } +/*[clinic input] +_ctypes.pointer as create_pointer_inst + + obj as arg: object + / + +Create a new pointer instance, pointing to 'obj'. + +The returned object is of the type POINTER(type(obj)). Note that if you +just want to pass a pointer to an object to a foreign function call, you +should use byref(obj) which is much faster. +[clinic start generated code]*/ + static PyObject * -pointer(PyObject *self, PyObject *arg) +create_pointer_inst(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=3b543bc9f0de2180 input=713685fdb4d9bc27]*/ { PyObject *result; PyObject *typ; @@ -1957,7 +1991,7 @@ pointer(PyObject *self, PyObject *arg) else if (PyErr_Occurred()) { return NULL; } - typ = POINTER(NULL, (PyObject *)Py_TYPE(arg)); + typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); if (typ == NULL) return NULL; result = PyObject_CallOneArg(typ, arg); @@ -1997,8 +2031,8 @@ buffer_info(PyObject *self, PyObject *arg) PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, - {"POINTER", POINTER, METH_O }, - {"pointer", pointer, METH_O }, + CREATE_POINTER_TYPE_METHODDEF + CREATE_POINTER_INST_METHODDEF {"_unpickle", unpickle, METH_VARARGS }, {"buffer_info", buffer_info, METH_O, "Return buffer interface information"}, {"resize", resize, METH_VARARGS, "Resize the memory buffer of a ctypes instance"}, diff --git a/Modules/_ctypes/clinic/callproc.c.h b/Modules/_ctypes/clinic/callproc.c.h new file mode 100644 index 00000000000000..6f036bb66b25aa --- /dev/null +++ b/Modules/_ctypes/clinic/callproc.c.h @@ -0,0 +1,38 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(create_pointer_type__doc__, +"POINTER($module, type, /)\n" +"--\n" +"\n" +"Create and return a new ctypes pointer type.\n" +"\n" +" type\n" +" A ctypes type.\n" +"\n" +"Pointer types are cached and reused internally,\n" +"so calling this function repeatedly is cheap."); + +#define CREATE_POINTER_TYPE_METHODDEF \ + {"POINTER", (PyCFunction)create_pointer_type, METH_O, create_pointer_type__doc__}, + +PyDoc_STRVAR(create_pointer_inst__doc__, +"pointer($module, obj, /)\n" +"--\n" +"\n" +"Create a new pointer instance, pointing to \'obj\'.\n" +"\n" +"The returned object is of the type POINTER(type(obj)). Note that if you\n" +"just want to pass a pointer to an object to a foreign function call, you\n" +"should use byref(obj) which is much faster."); + +#define CREATE_POINTER_INST_METHODDEF \ + {"pointer", (PyCFunction)create_pointer_inst, METH_O, create_pointer_inst__doc__}, +/*[clinic end generated code: output=ae26452a759ba56d input=a9049054013a1b77]*/ From 5df8b0d5c71a168a94fb64ad9d8190377b6e73da Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 8 Aug 2023 10:43:41 +0200 Subject: [PATCH 073/598] gh-95065: Argument Clinic: Add comment to preprocessor warning code (#107766) --- Lib/test/clinic.test.c | 27 ++++++++++++++++++--------- Tools/clinic/clinic.py | 1 + 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 386d1dac3f115e..9fcee0dad7118b 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5524,6 +5524,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * PyObject *a; PyObject *b = Py_None; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ @@ -5571,7 +5572,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * static PyObject * test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=6c0fd9d94fa1e765 input=89099f3dacd757da]*/ +/*[clinic end generated code: output=144cbf1adc574dd9 input=89099f3dacd757da]*/ /*[clinic input] @@ -5630,6 +5631,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ PyObject *a; PyObject *b; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ @@ -5669,7 +5671,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=22821a0fa9945d0c input=1702bbab1e9b3b99]*/ +/*[clinic end generated code: output=994bd57c1c634709 input=1702bbab1e9b3b99]*/ /*[clinic input] @@ -5735,6 +5737,7 @@ test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const * PyObject *c; PyObject *d; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ @@ -5781,7 +5784,7 @@ static PyObject * test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=061d554ccc6b8f51 input=28cdb885f6c34eab]*/ +/*[clinic end generated code: output=146c60ecbcdbf4b8 input=28cdb885f6c34eab]*/ /*[clinic input] @@ -5837,6 +5840,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ PyObject *argsbuf[1]; PyObject *a; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ @@ -5874,7 +5878,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=3e512117a5eda970 input=678206db25c0652c]*/ +/*[clinic end generated code: output=dce99971a2494f9f input=678206db25c0652c]*/ /*[clinic input] @@ -5933,6 +5937,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ PyObject *a; PyObject *b; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ @@ -5975,7 +5980,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=d41da050a5b82dd0 input=fae0d0b1d480c939]*/ +/*[clinic end generated code: output=06999692e0c8dac4 input=fae0d0b1d480c939]*/ /*[clinic input] @@ -6044,6 +6049,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *c; PyObject *e; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ @@ -6095,7 +6101,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, PyObject *b, PyObject *c, PyObject *e) -/*[clinic end generated code: output=c5d7ddfc139ddf31 input=1b0121770c0c52e0]*/ +/*[clinic end generated code: output=a553e33101dc42b2 input=1b0121770c0c52e0]*/ /*[clinic input] @@ -6156,6 +6162,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ PyObject *b; PyObject *c; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ @@ -6196,7 +6203,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=d80609e6b37ffb8a input=e1d129689e69ec7c]*/ +/*[clinic end generated code: output=f96454a4970b443c input=e1d129689e69ec7c]*/ /*[clinic input] @@ -6260,6 +6267,7 @@ test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ PyObject *c; PyObject *d; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ @@ -6305,7 +6313,7 @@ static PyObject * test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=1c10d6197562319f input=0d53533463a12792]*/ +/*[clinic end generated code: output=5e648e887da0a804 input=0d53533463a12792]*/ /*[clinic input] @@ -6377,6 +6385,7 @@ test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *d; PyObject *e; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ @@ -6427,4 +6436,4 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=d32375ffce63d3db input=154fd450448d8935]*/ +/*[clinic end generated code: output=383d56b03f7c2dcb input=154fd450448d8935]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 82e5b804c2e3ea..c6cf43ab40fb12 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -850,6 +850,7 @@ class CLanguage(Language): #endif /* !defined({methoddef_name}) */ """) DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" + // Emit compiler warnings when we get to Python {major}.{minor}. #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 # error \ {cpp_message} From 7c5153de5a2bd2c886173a317f116885a925cfce Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 8 Aug 2023 13:12:49 +0100 Subject: [PATCH 074/598] gh-106368: Argument clinic: add tests for more failure paths (#107731) --- Lib/test/test_clinic.py | 108 ++++++++++++++++++++++++++++++++++------ Tools/clinic/clinic.py | 5 +- 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index d13d8623f8093b..6c2411f9a57b62 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -45,6 +45,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): tc.assertEqual(cm.exception.filename, filename) if lineno is not None: tc.assertEqual(cm.exception.lineno, lineno) + return cm.exception class ClinicWholeFileTest(TestCase): @@ -222,6 +223,15 @@ def test_directive_output_print(self): last_line.startswith("/*[clinic end generated code: output=") ) + def test_directive_wrong_arg_number(self): + raw = dedent(""" + /*[clinic input] + preserve foo bar baz eggs spam ham mushrooms + [clinic start generated code]*/ + """) + err = "takes 1 positional argument but 8 were given" + self.expect_failure(raw, err) + def test_unknown_destination_command(self): raw = """ /*[clinic input] @@ -600,6 +610,31 @@ def test_directive_output_invalid_command(self): self.expect_failure(block, err, lineno=2) +class ParseFileUnitTest(TestCase): + def expect_parsing_failure( + self, *, filename, expected_error, verify=True, output=None + ): + errmsg = re.escape(dedent(expected_error).strip()) + with self.assertRaisesRegex(clinic.ClinicError, errmsg): + clinic.parse_file(filename) + + def test_parse_file_no_extension(self) -> None: + self.expect_parsing_failure( + filename="foo", + expected_error="Can't extract file type for file 'foo'" + ) + + def test_parse_file_strange_extension(self) -> None: + filenames_to_errors = { + "foo.rs": "Can't identify file type for file 'foo.rs'", + "foo.hs": "Can't identify file type for file 'foo.hs'", + "foo.js": "Can't identify file type for file 'foo.js'", + } + for filename, errmsg in filenames_to_errors.items(): + with self.subTest(filename=filename): + self.expect_parsing_failure(filename=filename, expected_error=errmsg) + + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): computed = clinic.permute_optional_groups(l, m, r) @@ -794,8 +829,8 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): return s[function_index] def expect_failure(self, block, err, *, filename=None, lineno=None): - _expect_failure(self, self.parse_function, block, err, - filename=filename, lineno=lineno) + return _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno) def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) @@ -877,6 +912,41 @@ def test_param_default_expr_named_constant(self): """ self.expect_failure(block, err, lineno=2) + def test_param_with_bizarre_default_fails_correctly(self): + template = """ + module os + os.access + follow_symlinks: int = {default} + """ + err = "Unsupported expression as default value" + for bad_default_value in ( + "{1, 2, 3}", + "3 if bool() else 4", + "[x for x in range(42)]" + ): + with self.subTest(bad_default=bad_default_value): + block = template.format(default=bad_default_value) + self.expect_failure(block, err, lineno=2) + + def test_unspecified_not_allowed_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = unspecified + """ + err = "'unspecified' is not a legal default value!" + exc = self.expect_failure(block, err, lineno=2) + self.assertNotIn('Malformed expression given as default value', str(exc)) + + def test_malformed_expression_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = 1/0 + """ + err = "Malformed expression given as default value" + self.expect_failure(block, err, lineno=2) + def test_param_default_expr_binop(self): err = ( "When you specify an expression ('a + b') as your default value, " @@ -1041,6 +1111,28 @@ def test_c_name(self): """) self.assertEqual("os_stat_fn", function.c_basename) + def test_base_invalid_syntax(self): + block = """ + module os + os.stat + invalid syntax: int = 42 + """ + err = dedent(r""" + Function 'stat' has an invalid parameter declaration: + \s+'invalid syntax: int = 42' + """).strip() + with self.assertRaisesRegex(clinic.ClinicError, err): + self.parse_function(block) + + def test_param_default_invalid_syntax(self): + block = """ + module os + os.stat + x: int = invalid syntax + """ + err = r"Syntax error: 'x = invalid syntax\n'" + self.expect_failure(block, err, lineno=2) + def test_cloning_nonexistent_function_correctly_fails(self): block = """ cloned = fooooooooooooooooo @@ -1414,18 +1506,6 @@ def test_parameters_required_after_star(self): with self.subTest(block=block): self.expect_failure(block, err) - def test_parameters_required_after_depr_star(self): - dataset = ( - "module foo\nfoo.bar\n * [from 3.14]", - "module foo\nfoo.bar\n * [from 3.14]\nDocstring here.", - "module foo\nfoo.bar\n this: int\n * [from 3.14]", - "module foo\nfoo.bar\n this: int\n * [from 3.14]\nDocstring.", - ) - err = "Function 'foo.bar' specifies '* [from 3.14]' without any parameters afterwards." - for block in dataset: - with self.subTest(block=block): - self.expect_failure(block, err) - def test_depr_star_invalid_format_1(self): block = """ module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c6cf43ab40fb12..0b336d9ac5a60f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5207,13 +5207,14 @@ def bad_node(self, node: ast.AST) -> None: # but at least make an attempt at ensuring it's a valid expression. try: value = eval(default) - if value is unspecified: - fail("'unspecified' is not a legal default value!") except NameError: pass # probably a named constant except Exception as e: fail("Malformed expression given as default value " f"{default!r} caused {e!r}") + else: + if value is unspecified: + fail("'unspecified' is not a legal default value!") if bad: fail(f"Unsupported expression as default value: {default!r}") From 906b73be5eada1995bd667a02c59f7a11998310f Mon Sep 17 00:00:00 2001 From: Fatih <77548106+fatihkabakk@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:47:15 +0300 Subject: [PATCH 075/598] gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) --- PCbuild/readme.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 86ad3ab1a40d9c..1d3b45b7912a34 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -251,9 +251,11 @@ against a profiling library and contain extra debug information. The PGUpdate configuration takes the profiling data and generates optimized binaries. -The build_pgo.bat script automates the creation of optimized binaries. -It creates the PGI files, runs the unit test suite or PyBench with the -PGI python, and finally creates the optimized files. +The build.bat script has an argument `--pgo` that automate the creation +of optimized binaries. +It creates the PGI files, runs the unit test suite with the PGI python, +and finally creates the optimized files. +You can customize the job for profiling with `--pgo-job ` option. See https://docs.microsoft.com/en-us/cpp/build/profile-guided-optimizations From f9e3ff1ea4b2c8b787360409d55f2037652b7456 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 8 Aug 2023 09:52:13 -0600 Subject: [PATCH 076/598] gh-105699: Re-enable the Multiple-Interpreters Stress Tests (gh-107572) We had disabled them due to crashes they exposed, which have since been fixed. --- Lib/test/test_interpreters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 662eafa3b7a104..5981d96de8de06 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -464,7 +464,6 @@ def test_bytes_for_script(self): # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. -@unittest.skip('these are crashing, likely just due just to _xxsubinterpreters (see gh-105699)') class StressTests(TestBase): # In these tests we generally want a lot of interpreters, From d4ac094cf9d15ec5705ec0fe8771df9e6ba915b9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 8 Aug 2023 18:12:52 +0200 Subject: [PATCH 077/598] Minor accuracy improvement for statistics.correlation() (GH-107781) --- Lib/statistics.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 6bd214bbfe2ff5..066669d25ddb12 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1004,6 +1004,14 @@ def _mean_stdev(data): # Handle Nans and Infs gracefully return float(xbar), float(xbar) / float(ss) +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with high accuracy." + # Square root differential correction: + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + h = sqrt(x * y) + x = sumprod((x, h), (y, -h)) + return h + x / (2.0 * h) + # === Statistics for relations between two inputs === @@ -1083,7 +1091,7 @@ def correlation(x, y, /, *, method='linear'): sxx = sumprod(x, x) syy = sumprod(y, y) try: - return sxy / sqrt(sxx * syy) + return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: raise StatisticsError('at least one of the inputs is constant') From aab6f7173a3b825599629dd6fa5cb7e477421595 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 8 Aug 2023 19:30:33 +0200 Subject: [PATCH 078/598] GH-100425: Note improved commutativity in sum(). (GH-107785) --- Doc/library/functions.rst | 2 +- Doc/whatsnew/3.12.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index b271067ae639c5..88a7fdfe6f0d50 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1752,7 +1752,7 @@ are always available. They are listed here in alphabetical order. The *start* parameter can be specified as a keyword argument. .. versionchanged:: 3.12 Summation of floats switched to an algorithm - that gives higher accuracy on most builds. + that gives higher accuracy and better commutativity on most builds. .. class:: super() diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 6553013552d003..8ed435476c9cec 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -504,8 +504,8 @@ Other Language Changes * :class:`slice` objects are now hashable, allowing them to be used as dict keys and set items. (Contributed by Will Bradshaw, Furkan Onder, and Raymond Hettinger in :gh:`101264`.) -* :func:`sum` now uses Neumaier summation to improve accuracy when summing - floats or mixed ints and floats. +* :func:`sum` now uses Neumaier summation to improve accuracy and commutativity + when summing floats or mixed ints and floats. (Contributed by Raymond Hettinger in :gh:`100425`.) * Exceptions raised in a typeobject's ``__set_name__`` method are no longer From ea72c6fe3b6db5f4e8ce3d3405c0ea65dc002faf Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 8 Aug 2023 13:42:43 -0700 Subject: [PATCH 079/598] GH-107596: Specialize str[int] (GH-107597) --- Include/internal/pycore_opcode.h | 36 +++--- Include/internal/pycore_opcode_metadata.h | 6 + Include/opcode.h | 103 +++++++++--------- Lib/_opcode_metadata.py | 1 + ...-08-03-11-13-09.gh-issue-107596.T3yPGI.rst | 1 + Python/bytecodes.c | 16 +++ Python/executor_cases.c.h | 23 ++++ Python/generated_cases.c.h | 24 ++++ Python/opcode_targets.h | 34 +++--- Python/specialize.c | 25 +++-- 10 files changed, 172 insertions(+), 97 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index aff09a2a926aec..a187da6e24730f 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -52,6 +52,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, [BUILD_LIST] = BUILD_LIST, @@ -292,12 +293,12 @@ const char *const _PyOpcode_OpName[268] = { [FORMAT_SIMPLE] = "FORMAT_SIMPLE", [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [SEND_GEN] = "SEND_GEN", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -305,39 +306,39 @@ const char *const _PyOpcode_OpName[268] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [RETURN_VALUE] = "RETURN_VALUE", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [LOAD_LOCALS] = "LOAD_LOCALS", - [COMPARE_OP_STR] = "COMPARE_OP_STR", + [COMPARE_OP_INT] = "COMPARE_OP_INT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -360,9 +361,9 @@ const char *const _PyOpcode_OpName[268] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [COMPARE_OP_STR] = "COMPARE_OP_STR", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -381,11 +382,11 @@ const char *const _PyOpcode_OpName[268] = { [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [RAISE_VARARGS] = "RAISE_VARARGS", [GET_AWAITABLE] = "GET_AWAITABLE", - [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [BUILD_SLICE] = "BUILD_SLICE", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [MAKE_CELL] = "MAKE_CELL", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_DEREF] = "LOAD_DEREF", [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", @@ -397,26 +398,26 @@ const char *const _PyOpcode_OpName[268] = { [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", [MAP_ADD] = "MAP_ADD", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [COPY_FREE_VARS] = "COPY_FREE_VARS", [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", [CONVERT_VALUE] = "CONVERT_VALUE", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", @@ -427,6 +428,7 @@ const char *const _PyOpcode_OpName[268] = { [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", @@ -435,7 +437,6 @@ const char *const _PyOpcode_OpName[268] = { [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", - [186] = "<186>", [187] = "<187>", [188] = "<188>", [189] = "<189>", @@ -521,7 +522,6 @@ const char *const _PyOpcode_OpName[268] = { #endif // NEED_OPCODE_TABLES #define EXTRA_CASES \ - case 186: \ case 187: \ case 188: \ case 189: \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 1cab6c984f3ace..02303c42c75c8a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -144,6 +144,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 4; case BINARY_SUBSCR_LIST_INT: return 2; + case BINARY_SUBSCR_STR_INT: + return 2; case BINARY_SUBSCR_TUPLE_INT: return 2; case BINARY_SUBSCR_DICT: @@ -588,6 +590,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case BINARY_SUBSCR_LIST_INT: return 1; + case BINARY_SUBSCR_STR_INT: + return 1; case BINARY_SUBSCR_TUPLE_INT: return 1; case BINARY_SUBSCR_DICT: @@ -1047,6 +1051,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, 0 }, @@ -1258,6 +1263,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, + [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_STR_INT, 0, 0 } } }, [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, diff --git a/Include/opcode.h b/Include/opcode.h index ede1518b6fd25c..b3d6cba63096c8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -175,57 +175,58 @@ extern "C" { #define BINARY_SUBSCR_DICT 38 #define BINARY_SUBSCR_GETITEM 39 #define BINARY_SUBSCR_LIST_INT 42 -#define BINARY_SUBSCR_TUPLE_INT 43 -#define STORE_SUBSCR_DICT 44 -#define STORE_SUBSCR_LIST_INT 45 -#define SEND_GEN 46 -#define UNPACK_SEQUENCE_TWO_TUPLE 47 -#define UNPACK_SEQUENCE_TUPLE 48 -#define UNPACK_SEQUENCE_LIST 56 -#define STORE_ATTR_INSTANCE_VALUE 57 -#define STORE_ATTR_SLOT 58 -#define STORE_ATTR_WITH_HINT 59 -#define LOAD_GLOBAL_MODULE 62 -#define LOAD_GLOBAL_BUILTIN 63 -#define LOAD_SUPER_ATTR_ATTR 64 -#define LOAD_SUPER_ATTR_METHOD 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_WITH_HINT 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_CLASS 73 -#define LOAD_ATTR_PROPERTY 76 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_ATTR_METHOD_NO_DICT 79 -#define LOAD_ATTR_METHOD_LAZY_DICT 80 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 81 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 82 -#define COMPARE_OP_FLOAT 84 -#define COMPARE_OP_INT 86 -#define COMPARE_OP_STR 88 -#define FOR_ITER_LIST 111 -#define FOR_ITER_TUPLE 112 -#define FOR_ITER_RANGE 113 -#define FOR_ITER_GEN 132 -#define CALL_BOUND_METHOD_EXACT_ARGS 136 -#define CALL_PY_EXACT_ARGS 148 -#define CALL_PY_WITH_DEFAULTS 153 -#define CALL_NO_KW_TYPE_1 154 -#define CALL_NO_KW_STR_1 155 -#define CALL_NO_KW_TUPLE_1 159 -#define CALL_BUILTIN_CLASS 160 -#define CALL_NO_KW_BUILTIN_O 161 -#define CALL_NO_KW_BUILTIN_FAST 166 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 -#define CALL_NO_KW_LEN 178 -#define CALL_NO_KW_ISINSTANCE 179 -#define CALL_NO_KW_LIST_APPEND 180 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 181 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 182 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 183 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 184 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 185 +#define BINARY_SUBSCR_STR_INT 43 +#define BINARY_SUBSCR_TUPLE_INT 44 +#define STORE_SUBSCR_DICT 45 +#define STORE_SUBSCR_LIST_INT 46 +#define SEND_GEN 47 +#define UNPACK_SEQUENCE_TWO_TUPLE 48 +#define UNPACK_SEQUENCE_TUPLE 56 +#define UNPACK_SEQUENCE_LIST 57 +#define STORE_ATTR_INSTANCE_VALUE 58 +#define STORE_ATTR_SLOT 59 +#define STORE_ATTR_WITH_HINT 62 +#define LOAD_GLOBAL_MODULE 63 +#define LOAD_GLOBAL_BUILTIN 64 +#define LOAD_SUPER_ATTR_ATTR 65 +#define LOAD_SUPER_ATTR_METHOD 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 70 +#define LOAD_ATTR_WITH_HINT 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_CLASS 76 +#define LOAD_ATTR_PROPERTY 77 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_LAZY_DICT 81 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 +#define COMPARE_OP_FLOAT 86 +#define COMPARE_OP_INT 88 +#define COMPARE_OP_STR 111 +#define FOR_ITER_LIST 112 +#define FOR_ITER_TUPLE 113 +#define FOR_ITER_RANGE 132 +#define FOR_ITER_GEN 136 +#define CALL_BOUND_METHOD_EXACT_ARGS 148 +#define CALL_PY_EXACT_ARGS 153 +#define CALL_PY_WITH_DEFAULTS 154 +#define CALL_NO_KW_TYPE_1 155 +#define CALL_NO_KW_STR_1 159 +#define CALL_NO_KW_TUPLE_1 160 +#define CALL_BUILTIN_CLASS 161 +#define CALL_NO_KW_BUILTIN_O 166 +#define CALL_NO_KW_BUILTIN_FAST 167 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 +#define CALL_NO_KW_LEN 179 +#define CALL_NO_KW_ISINSTANCE 180 +#define CALL_NO_KW_LIST_APPEND 181 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 #define NB_ADD 0 #define NB_AND 1 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index fd8ecdb5c980f3..17101d1d94757b 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -25,6 +25,7 @@ "BINARY_SUBSCR_DICT", "BINARY_SUBSCR_GETITEM", "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_STR_INT", "BINARY_SUBSCR_TUPLE_INT", ], "STORE_SUBSCR": [ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst new file mode 100644 index 00000000000000..8912de73680b44 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst @@ -0,0 +1 @@ +Specialize subscripting :class:`str` objects by :class:`int` indexes. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 90e26d3c86b380..d6bfb624c77133 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -509,6 +509,7 @@ dummy_func( BINARY_SUBSCR_DICT, BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, + BINARY_SUBSCR_STR_INT, BINARY_SUBSCR_TUPLE_INT, }; @@ -574,6 +575,21 @@ dummy_func( Py_DECREF(list); } + inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + } + inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9363b4955087db..27be8a383989ee 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -462,6 +462,29 @@ break; } + case BINARY_SUBSCR_STR_INT: { + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + case BINARY_SUBSCR_TUPLE_INT: { PyObject *sub; PyObject *tuple; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7250240ac2396c..d7db8b07005fd1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -685,6 +685,30 @@ DISPATCH(); } + TARGET(BINARY_SUBSCR_STR_INT) { + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; + DISPATCH(); + } + TARGET(BINARY_SUBSCR_TUPLE_INT) { PyObject *sub; PyObject *tuple; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index d84d253c912a28..210c37b37225bb 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -42,12 +42,12 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, &&TARGET_BINARY_SUBSCR_LIST_INT, + &&TARGET_BINARY_SUBSCR_STR_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_SEND_GEN, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_SUPER_ATTR_ATTR, &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_RETURN_VALUE, - &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_COMPARE_OP_INT, + &&TARGET_COMPARE_OP_FLOAT, &&TARGET_LOAD_LOCALS, - &&TARGET_COMPARE_OP_STR, + &&TARGET_COMPARE_OP_INT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_COMPARE_OP_STR, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, - &&TARGET_FOR_ITER_RANGE, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -131,11 +131,11 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, &&TARGET_GET_AWAITABLE, - &&TARGET_FOR_ITER_GEN, + &&TARGET_FOR_ITER_RANGE, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_MAKE_CELL, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, @@ -147,26 +147,26 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, - &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_COPY_FREE_VARS, &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_CALL_NO_KW_STR_1, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, &&TARGET_CONVERT_VALUE, + &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_BUILTIN_FAST, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_LOAD_FAST_LOAD_FAST, &&TARGET_STORE_FAST_LOAD_FAST, &&TARGET_STORE_FAST_STORE_FAST, @@ -177,6 +177,7 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, &&TARGET_LOAD_FROM_DICT_OR_DEREF, &&TARGET_SET_FUNCTION_ATTRIBUTE, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_LEN, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LIST_APPEND, @@ -228,7 +229,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_ENTER_EXECUTOR, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index de329ef1195cbf..855252e066dea1 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -363,7 +363,6 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10 #define SPEC_FAIL_SUBSCR_LIST_SLICE 11 #define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12 -#define SPEC_FAIL_SUBSCR_STRING_INT 13 #define SPEC_FAIL_SUBSCR_STRING_SLICE 14 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16 @@ -1260,16 +1259,7 @@ _Py_Specialize_LoadGlobal( static int binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub) { - if (container_type == &PyUnicode_Type) { - if (PyLong_CheckExact(sub)) { - return SPEC_FAIL_SUBSCR_STRING_INT; - } - if (PySlice_Check(sub)) { - return SPEC_FAIL_SUBSCR_STRING_SLICE; - } - return SPEC_FAIL_OTHER; - } - else if (strcmp(container_type->tp_name, "array.array") == 0) { + if (strcmp(container_type->tp_name, "array.array") == 0) { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_ARRAY_INT; } @@ -1376,6 +1366,19 @@ _Py_Specialize_BinarySubscr( PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER); goto fail; } + if (container_type == &PyUnicode_Type) { + if (PyLong_CheckExact(sub)) { + if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + instr->op.code = BINARY_SUBSCR_STR_INT; + goto success; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, + PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_STRING_SLICE : SPEC_FAIL_OTHER); + goto fail; + } if (container_type == &PyDict_Type) { instr->op.code = BINARY_SUBSCR_DICT; goto success; From 0be3743f54569475c40e00c8dcdded928d429d58 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 8 Aug 2023 22:50:54 +0200 Subject: [PATCH 080/598] gh-104683: Add --exclude option to Argument Clinic CLI (#107770) Co-authored-by: Serhiy Storchaka --- Doc/howto/clinic.rst | 5 ++++ Lib/test/test_clinic.py | 29 +++++++++++++++++++ Makefile.pre.in | 6 +++- ...-08-08-12-21-41.gh-issue-104683.DRsAQE.rst | 1 + Tools/clinic/clinic.py | 11 +++++++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 743c7c9cb3000e..2d89ccc203b6d6 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -188,6 +188,11 @@ The CLI supports the following options: The directory tree to walk in :option:`--make` mode. +.. option:: --exclude EXCLUDE + + A file to exclude in :option:`--make` mode. + This option can be given multiple times. + .. option:: FILE ... The list of files to process. diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6c2411f9a57b62..8ed7e214742d50 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2216,6 +2216,35 @@ def create_files(files, srcdir, code): path = os.path.join(ext_path, filename) self.assertNotIn(path, out) + def test_cli_make_exclude(self): + code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + """) + with os_helper.temp_dir(quiet=False) as tmp_dir: + # add some folders, some C files and a Python file + for fn in "file1.c", "file2.c", "file3.c", "file4.c": + path = os.path.join(tmp_dir, fn) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + + # Run clinic in verbose mode with --make on tmpdir. + # Exclude file2.c and file3.c. + out = self.expect_success( + "-v", "--make", "--srcdir", tmp_dir, + "--exclude", os.path.join(tmp_dir, "file2.c"), + # The added ./ should be normalised away. + "--exclude", os.path.join(tmp_dir, "./file3.c"), + # Relative paths should also work. + "--exclude", "file4.c" + ) + + # expect verbose mode to only mention the C files in tmp_dir + self.assertIn("file1.c", out) + self.assertNotIn("file2.c", out) + self.assertNotIn("file3.c", out) + self.assertNotIn("file4.c", out) + def test_cli_verbose(self): with os_helper.temp_dir() as tmp_dir: fn = os.path.join(tmp_dir, "test.c") diff --git a/Makefile.pre.in b/Makefile.pre.in index 12409774746a30..d8fdb34747011d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -775,9 +775,13 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py +.PHONY: clinic-tests +clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $(srcdir)/Lib/test/clinic.test.c + # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst new file mode 100644 index 00000000000000..ee3a70967b098b --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst @@ -0,0 +1 @@ +Add ``--exclude`` option to Argument Clinic CLI. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0b336d9ac5a60f..3b26a706bafd32 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5834,6 +5834,9 @@ def create_cli() -> argparse.ArgumentParser: help="walk --srcdir to run over all relevant files") cmdline.add_argument("--srcdir", type=str, default=os.curdir, help="the directory tree to walk in --make mode") + cmdline.add_argument("--exclude", type=str, action="append", + help=("a file to exclude in --make mode; " + "can be given multiple times")) cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", help="the list of files to process") return cmdline @@ -5905,6 +5908,11 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: parser.error("can't use -o or filenames with --make") if not ns.srcdir: parser.error("--srcdir must not be empty with --make") + if ns.exclude: + excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] + excludes = [os.path.normpath(f) for f in excludes] + else: + excludes = [] for root, dirs, files in os.walk(ns.srcdir): for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): if rcs_dir in dirs: @@ -5914,6 +5922,9 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: if not filename.endswith(('.c', '.cpp', '.h')): continue path = os.path.join(root, filename) + path = os.path.normpath(path) + if path in excludes: + continue if ns.verbose: print(path) parse_file(path, verify=not ns.force) From 73507382ac184a72b59ebb0c2f85e8b1d2dfa58e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 8 Aug 2023 23:12:35 +0200 Subject: [PATCH 081/598] gh-104683: Argument Clinic: refactor format_docstring() (#107623) Extract helper methods for formatting the signature and parameter sections, and clean up the remaining function body. Co-authored-by: Alex Waygood Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Tools/clinic/clinic.py | 82 +++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3b26a706bafd32..16f31717080e1f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5506,23 +5506,11 @@ def state_function_docstring(self, line: str) -> None: self.docstring_append(self.function, line) - def format_docstring(self) -> str: - f = self.function - assert f is not None - - new_or_init = f.kind.new_or_init - if new_or_init and not f.docstring: - # don't render a docstring at all, no signature, nothing. - return f.docstring - + def format_docstring_signature( + self, f: Function, parameters: list[Parameter] + ) -> str: text, add, output = _text_accumulator() - parameters = f.render_parameters - - ## - ## docstring first line - ## - - if new_or_init: + if f.kind.new_or_init: # classes get *just* the name of the class # not __new__, not __init__, and not module.classname assert f.cls @@ -5685,35 +5673,39 @@ def add_parameter(text: str) -> None: if not f.docstring_only: add("\n" + sig_end_marker + "\n") - docstring_first_line = output() + signature_line = output() # now fix up the places where the brackets look wrong - docstring_first_line = docstring_first_line.replace(', ]', ',] ') + return signature_line.replace(', ]', ',] ') - # okay. now we're officially building the "parameters" section. - # create substitution text for {parameters} + @staticmethod + def format_docstring_parameters(params: list[Parameter]) -> str: + """Create substitution text for {parameters}""" + text, add, output = _text_accumulator() spacer_line = False - for p in parameters: - if not p.docstring.strip(): + for param in params: + docstring = param.docstring.strip() + if not docstring: continue if spacer_line: add('\n') else: spacer_line = True add(" ") - add(p.name) + add(param.name) add('\n') - add(textwrap.indent(rstrip_lines(p.docstring.rstrip()), " ")) - parameters_output = output() - if parameters_output: - parameters_output += '\n' - - ## - ## docstring body - ## + stripped = rstrip_lines(docstring) + add(textwrap.indent(stripped, " ")) + if text: + add('\n') + return output() - docstring = f.docstring.rstrip() - lines = [line.rstrip() for line in docstring.split('\n')] + def format_docstring(self) -> str: + assert self.function is not None + f = self.function + if f.kind.new_or_init and not f.docstring: + # don't render a docstring at all, no signature, nothing. + return f.docstring # Enforce the summary line! # The first line of a docstring should be a summary of the function. @@ -5727,6 +5719,7 @@ def add_parameter(text: str) -> None: # Guido said Clinic should enforce this: # http://mail.python.org/pipermail/python-dev/2013-June/127110.html + lines = f.docstring.split('\n') if len(lines) >= 2: if lines[1]: fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" @@ -5738,26 +5731,23 @@ def add_parameter(text: str) -> None: # between it and the {parameters} we're about to add. lines.append('') - parameters_marker_count = len(docstring.split('{parameters}')) - 1 + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 if parameters_marker_count > 1: fail('You may not specify {parameters} more than once in a docstring!') + # insert signature at front and params after the summary line if not parameters_marker_count: - # insert after summary line lines.insert(2, '{parameters}') + lines.insert(0, '{signature}') - # insert at front of docstring - lines.insert(0, docstring_first_line) - + # finalize docstring + params = f.render_parameters + parameters = self.format_docstring_parameters(params) + signature = self.format_docstring_signature(f, params) docstring = "\n".join(lines) - - add(docstring) - docstring = output() - - docstring = linear_format(docstring, parameters=parameters_output) - docstring = docstring.rstrip() - - return docstring + return linear_format(docstring, + signature=signature, + parameters=parameters).rstrip() def do_post_block_processing_cleanup(self, lineno: int) -> None: """ From 7b6e34e5baeb4162815ffa4d943b09a58e3f6580 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 9 Aug 2023 08:47:57 +0300 Subject: [PATCH 082/598] gh-106052: Fix bug in the matching of possessive quantifiers (gh-106515) It did not work in the case of a subpattern containing backtracking. Temporary implement possessive quantifiers as equivalent greedy qualifiers in atomic groups. --- Lib/re/_compiler.py | 7 +++++++ Lib/test/test_re.py | 12 ++++++++++++ .../2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst | 2 ++ 3 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index d0a4c55caf6e41..f5fd160ba00435 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,6 +100,13 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) + elif op is POSSESSIVE_REPEAT: + # gh-106052: Possessive quantifiers do not work when the + # subpattern contains backtracking, i.e. "(?:ab?c)*+". + # Implement it as equivalent greedy qualifier in atomic group. + p = [(MAX_REPEAT, av)] + p = [(ATOMIC_GROUP, p)] + _compile(code, p, flags) elif op in REPEATING_CODES: if _simple(av[2]): emit(REPEATING_CODES[op][2]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a565cbe1a8d810..bf3698ac78a880 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2342,6 +2342,16 @@ def test_bug_gh91616(self): self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer self.assertTrue(re.fullmatch(r'(?s:(?=(?P.*?\.))(?P=g0).*)\Z', "a.txt")) + def test_bug_gh106052(self): + self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)*+", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)?)", "a").span(), (0, 0)) + self.assertEqual(re.match("(?:ab?c)?+", "a").span(), (0, 0)) + self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2441,6 +2451,7 @@ def test_atomic_group(self): 17: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2453,6 +2464,7 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst new file mode 100644 index 00000000000000..f2d4c2f7b18ec7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst @@ -0,0 +1,2 @@ +:mod:`re` module: fix the matching of possessive quantifiers in the case of +a subpattern containing backtracking. From 925bbc2166d169962210314772d13270e9ba60a2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 9 Aug 2023 07:55:34 +0200 Subject: [PATCH 083/598] gh-80282: Argument Clinic: Add clarifying comment about ASCII docstring limitation (#107764) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 16f31717080e1f..059c2db27288db 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5461,6 +5461,11 @@ def state_parameter_docstring_start(self, line: str) -> None: def docstring_append(self, obj: Function | Parameter, line: str) -> None: """Add a rstripped line to the current docstring.""" + # gh-80282: We filter out non-ASCII characters from the docstring, + # since historically, some compilers may balk on non-ASCII input. + # If you're using Argument Clinic in an external project, + # you may not need to support the same array of platforms as CPython, + # so you may be able to remove this restriction. matches = re.finditer(r'[^\x00-\x7F]', line) if offending := ", ".join([repr(m[0]) for m in matches]): warn("Non-ascii characters are not allowed in docstrings:", From 34cafd35e35dcb44b4347a6478fdbf88b900240c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 9 Aug 2023 09:43:02 +0200 Subject: [PATCH 084/598] Docs: clean up Argument Clinic howto's (#107797) - fix formatting in @text_signature howto and NEWS entry - fix formatting and example text in deprecate-positionals howto --- Doc/howto/clinic.rst | 21 ++++++++++--------- ...3-08-07-16-30-48.gh-issue-95065.-im4R5.rst | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 2d89ccc203b6d6..463a938fafa8dc 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1930,13 +1930,14 @@ Example from :source:`Objects/codeobject.c`:: Return a copy of the code object with new values for the specified fields. [clinic start generated output]*/ -The generated docstring ends up looking like this:: +The generated docstring ends up looking like this: - Doc_STRVAR(code_replace__doc__, - "replace($self, /, **changes)\n" - "--\n" - "\n" - "Return a copy of the code object with new values for the specified fields."); +.. code-block:: none + + replace($self, /, **changes) + -- + + Return a copy of the code object with new values for the specified fields. .. _clinic-howto-deprecate-positional: @@ -1968,7 +1969,7 @@ whenever the *b* parameter is passed positionally, and we'll be allowed to make it keyword-only in Python 3.14 at the earliest. We can use Argument Clinic to emit the desired deprecation warnings -using the ``* [from ...]``` syntax, +using the ``* [from ...]`` syntax, by adding the line ``* [from 3.14]`` right above the *b* parameter:: /*[clinic input] @@ -2000,12 +2001,12 @@ Luckily for us, compiler warnings are now generated: .. code-block:: none In file included from Modules/foomodule.c:139: - Modules/clinic/foomodule.c.h:83:8: warning: Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only. [-W#warnings] - # warning "Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only." + Modules/clinic/foomodule.c.h:139:8: warning: In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings] + # warning "In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings]" ^ We now close the deprecation phase by making *b* keyword-only; -replace the ``* [from ...]``` line above *b* +replace the ``* [from ...]`` line above *b* with the ``*`` from the line above *c*:: /*[clinic input] diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst index df1893e9c808ef..7284f5bd548810 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -1,2 +1,2 @@ Argument Clinic now supports overriding automatically generated signature by -using directive `@text_signature`. See :ref:`clinic-howto-override-signature`. +using directive ``@text_signature``. See :ref:`clinic-howto-override-signature`. From 2fb484e62518d15fc9a19565c2ab096bd741219a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 9 Aug 2023 08:44:43 +0100 Subject: [PATCH 085/598] Future-proof helper function with zero handling. (GH-107798) --- Lib/statistics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/statistics.py b/Lib/statistics.py index 066669d25ddb12..93c44f1f13fab7 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1009,6 +1009,8 @@ def _sqrtprod(x: float, y: float) -> float: # Square root differential correction: # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 h = sqrt(x * y) + if not h: + return 0.0 x = sumprod((x, h), (y, -h)) return h + x / (2.0 * h) From 52fbcf61b5a70993c2d32332ff0ad9f369d968d3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 9 Aug 2023 09:30:50 +0100 Subject: [PATCH 086/598] GH-107724: Fix the signature of `PY_THROW` callback functions. (GH-107725) --- Include/internal/pycore_instruments.h | 4 --- Lib/test/test_monitoring.py | 32 +++++++++++++++++++ ...-08-04-21-25-26.gh-issue-107724.EbBXMr.rst | 3 ++ Python/ceval.c | 2 +- Python/instrumentation.c | 10 ------ Python/legacy_tracing.c | 6 ++-- 6 files changed, 39 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index ccccd54a2f70a2..56de9f87171484 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -90,10 +90,6 @@ extern int _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void -_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); - extern void _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index b36382c8d0982d..845185be737eb2 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -743,6 +743,13 @@ class ExceptionHandledRecorder(ExceptionRecorder): def __call__(self, code, offset, exc): self.events.append(("handled", type(exc))) +class ThrowRecorder(ExceptionRecorder): + + event_type = E.PY_THROW + + def __call__(self, code, offset, exc): + self.events.append(("throw", type(exc))) + class ExceptionMonitoringTest(CheckEvents): @@ -888,6 +895,31 @@ async def async_loop(): func, recorders = self.exception_recorders) + def test_throw(self): + + def gen(): + yield 1 + yield 2 + + def func(): + try: + g = gen() + next(g) + g.throw(IndexError) + except IndexError: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + events = self.get_events( + func, + TEST_TOOL, + self.exception_recorders + (ThrowRecorder,) + ) + self.assertEqual(events[0], ("throw", IndexError)) + class LineRecorder: event_type = E.LINE diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst new file mode 100644 index 00000000000000..6e853cf72a3348 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst @@ -0,0 +1,3 @@ +In pre-release versions of 3.12, up to rc1, the sys.monitoring callback +function for the ``PY_THROW`` event was missing the third, exception +argument. That is now fixed. diff --git a/Python/ceval.c b/Python/ceval.c index 1dc4fd80224bbd..b966399a342d08 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2039,7 +2039,7 @@ monitor_throw(PyThreadState *tstate, if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } - _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); } void diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 123c20dfe1a99b..65ea7902a80a60 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1081,16 +1081,6 @@ call_instrumentation_vector_protected( assert(_PyErr_Occurred(tstate)); } -void -_Py_call_instrumentation_exc0( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) -{ - assert(_PyErr_Occurred(tstate)); - PyObject *args[3] = { NULL, NULL, NULL }; - call_instrumentation_vector_protected(tstate, event, frame, instr, 2, args); -} - void _Py_call_instrumentation_exc2( PyThreadState *tstate, int event, diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 48db51731cfdb0..7774d10b10172b 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -163,7 +163,7 @@ sys_trace_func2( } static PyObject * -sys_trace_unwind( +sys_trace_func3( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -445,7 +445,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + (vectorcallfunc)sys_trace_func3, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } @@ -470,7 +470,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_unwind, PyTrace_RETURN, + (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } From 65ce3652fa47a34acf715ee07bf0a2fae2e0da51 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 9 Aug 2023 11:24:05 +0100 Subject: [PATCH 087/598] gh-104683: Remove unused variables from `Tools/clinic` and tests for `Tools/clinic` (#107771) --- Lib/test/test_clinic.py | 9 ++------- Tools/clinic/cpp.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 8ed7e214742d50..026cd18d646ad7 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -567,7 +567,6 @@ def test_directive_preserve_input(self): self.expect_failure(block, err, lineno=6) def test_directive_preserve_output(self): - err = "'preserve' only works for blocks that don't produce any output!" block = dedent(""" /*[clinic input] output everything buffer @@ -965,7 +964,6 @@ def test_param_no_docstring(self): follow_symlinks: bool = True something_else: str = '' """) - p = function.parameters['follow_symlinks'] self.assertEqual(3, len(function.parameters)) conv = function.parameters['something_else'].converter self.assertIsInstance(conv, clinic.str_converter) @@ -1929,10 +1927,6 @@ def test_scaffolding(self): self.assertEqual(repr(clinic.NULL), '') # test that fail fails - expected = ( - 'Error in file "clown.txt" on line 69:\n' - 'The igloos are melting!\n' - ) with support.captured_stdout() as stdout: errmsg = 'The igloos are melting' with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: @@ -1940,6 +1934,7 @@ def test_scaffolding(self): exc = cm.exception self.assertEqual(exc.filename, 'clown.txt') self.assertEqual(exc.lineno, 69) + self.assertEqual(stdout.getvalue(), "") def test_non_ascii_character_in_docstring(self): block = """ @@ -2408,7 +2403,7 @@ def test_file_dest(self): "not overwriting!") self.assertIn(expected_err, err) # Run clinic again, this time with the -f option. - out = self.expect_success("-f", in_fn) + _ = self.expect_success("-f", in_fn) # Read back the generated output. with open(in_fn, encoding="utf-8") as f: data = f.read() diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 3d9c61ac678965..16eee6fc399491 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -186,7 +186,7 @@ def _main(filenames: list[str] | None = None) -> None: cpp = Monitor(filename, verbose=True) print() print(filename) - for line_number, line in enumerate(f.read().split('\n'), 1): + for line in f: cpp.writeline(line) From 1b3f5f24af95c0476ba339ed786a8a4a2578362a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 9 Aug 2023 14:44:26 +0200 Subject: [PATCH 088/598] gh-104683: Argument Clinic: Params now render their own docstrings (#107790) Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 12 ++++++++++-- Tools/clinic/clinic.py | 27 +++++++++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 026cd18d646ad7..3921523af067f8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -833,8 +833,8 @@ def expect_failure(self, block, err, *, filename=None, lineno=None): def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) - self.assertEqual(fn.docstring.strip(), - dedent(expected).strip()) + self.assertEqual(dedent(expected).strip(), + fn.docstring.strip()) def test_trivial(self): parser = DSLParser(_make_clinic()) @@ -997,8 +997,12 @@ def test_function_docstring(self): path: str Path to be examined + Ensure that multiple lines are indented correctly. Perform a stat system call on the given path. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. """) self.checkDocstring(function, """ stat($module, /, path) @@ -1008,6 +1012,10 @@ def test_function_docstring(self): path Path to be examined + Ensure that multiple lines are indented correctly. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. """) def test_docstring_trailing_whitespace(self): diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 059c2db27288db..3490d4871b8912 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2775,6 +2775,13 @@ def get_displayname(self, i: int) -> str: else: return f'"argument {i}"' + def render_docstring(self) -> str: + add, out = text_accumulator() + add(f" {self.name}\n") + for line in self.docstring.split("\n"): + add(f" {line}\n") + return out().rstrip() + CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) @@ -5686,23 +5693,11 @@ def add_parameter(text: str) -> None: @staticmethod def format_docstring_parameters(params: list[Parameter]) -> str: """Create substitution text for {parameters}""" - text, add, output = _text_accumulator() - spacer_line = False - for param in params: - docstring = param.docstring.strip() - if not docstring: - continue - if spacer_line: + add, output = text_accumulator() + for p in params: + if p.docstring: + add(p.render_docstring()) add('\n') - else: - spacer_line = True - add(" ") - add(param.name) - add('\n') - stripped = rstrip_lines(docstring) - add(textwrap.indent(stripped, " ")) - if text: - add('\n') return output() def format_docstring(self) -> str: From 0a7f48b9a8d54809f1e9272337aefe2444158e64 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 9 Aug 2023 15:28:18 +0200 Subject: [PATCH 089/598] gh-95065: Produce nicer deprecation messages in Argument Clinic (#107808) --- Lib/test/clinic.test.c | 193 ++++++++++++++++++++++++++++++++++++++++- Tools/clinic/clinic.py | 36 ++++---- 2 files changed, 211 insertions(+), 18 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 9fcee0dad7118b..c7063e1139b0a9 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4,9 +4,11 @@ output preset block /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3c81ac2402d06a8b]*/ /*[clinic input] +module m +class m.T "TestObj *" "TestType" class Test "TestObj *" "TestType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=fc7e50384d12b83f]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f761b4d55cb179cf]*/ /*[clinic input] test_object_converter @@ -6437,3 +6439,192 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *d, PyObject *e) /*[clinic end generated code: output=383d56b03f7c2dcb input=154fd450448d8935]*/ + + +/*[clinic input] +@classmethod +Test.__new__ + * [from 3.14] + a: object +The deprecation message should use the class name instead of __new__. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__doc__, +"Test(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing positional arguments to Test() is deprecated. Parameter\n" +"\'a\' will become a keyword-only parameter in Python 3.14.\n" +""); + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *a); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Test", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'Test.__new__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'Test.__new__' to be keyword-only.") + # else + # warning \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'Test.__new__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to Test() is deprecated. Parameter " + "'a' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = Test_impl(type, a); + +exit: + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=d15a69ea37ec6502 input=f133dc077aef49ec]*/ + + +/*[clinic input] +m.T.__init__ + * [from 3.14] + a: object +The deprecation message should use the class name instead of __init__. +[clinic start generated code]*/ + +PyDoc_STRVAR(m_T___init____doc__, +"T(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing positional arguments to m.T() is deprecated. Parameter\n" +"\'a\' will become a keyword-only parameter in Python 3.14.\n" +""); + +static int +m_T___init___impl(TestObj *self, PyObject *a); + +static int +m_T___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "T", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'm.T.__init__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'm.T.__init__' to be keyword-only.") + # else + # warning \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'm.T.__init__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to m.T() is deprecated. Parameter " + "'a' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = m_T___init___impl((TestObj *)self, a); + +exit: + return return_value; +} + +static int +m_T___init___impl(TestObj *self, PyObject *a) +/*[clinic end generated code: output=ef43c425816a549f input=f71b51dbe19fa657]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3490d4871b8912..70b066cce82fae 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -928,7 +928,7 @@ def deprecate_positional_use( if first_pos: preamble = f"Passing {first_pos+1} positional arguments to " depr_message = preamble + ( - f"{func.full_name}() is deprecated. Parameter {pstr} will " + f"{func.fulldisplayname}() is deprecated. Parameter {pstr} will " f"become a keyword-only parameter in Python {major}.{minor}." ) else: @@ -939,7 +939,7 @@ def deprecate_positional_use( f"argument{'s' if first_pos != 1 else ''} to " ) depr_message = preamble + ( - f"{func.full_name}() is deprecated. Parameters {pstr} will " + f"{func.fulldisplayname}() is deprecated. Parameters {pstr} will " f"become keyword-only parameters in Python {major}.{minor}." ) @@ -1673,14 +1673,7 @@ def render_function( full_name = f.full_name template_dict = {'full_name': full_name} - - if new_or_init: - assert isinstance(f.cls, Class) - name = f.cls.name - else: - name = f.name - - template_dict['name'] = name + template_dict['name'] = f.displayname if f.c_basename: c_basename = f.c_basename @@ -2678,6 +2671,21 @@ def __post_init__(self) -> None: self.self_converter: self_converter | None = None self.__render_parameters__: list[Parameter] | None = None + @functools.cached_property + def displayname(self) -> str: + """Pretty-printable name.""" + if self.kind.new_or_init: + assert isinstance(self.cls, Class) + return self.cls.name + else: + return self.name + + @functools.cached_property + def fulldisplayname(self) -> str: + if isinstance(self.module, Module): + return f"{self.module.name}.{self.displayname}" + return self.displayname + @property def render_parameters(self) -> list[Parameter]: if not self.__render_parameters__: @@ -5522,13 +5530,7 @@ def format_docstring_signature( self, f: Function, parameters: list[Parameter] ) -> str: text, add, output = _text_accumulator() - if f.kind.new_or_init: - # classes get *just* the name of the class - # not __new__, not __init__, and not module.classname - assert f.cls - add(f.cls.name) - else: - add(f.name) + add(f.displayname) if self.forced_text_signature: add(self.forced_text_signature) else: From a9caf9cf9041d6d0b69f8be0fd778dd1f9b50e74 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 9 Aug 2023 11:19:39 -0700 Subject: [PATCH 090/598] GH-105848: Simplify the arrangement of CALL's stack (GH-107788) --- Doc/library/dis.rst | 2 +- Include/internal/pycore_opcode_metadata.h | 18 +- Lib/dis.py | 6 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/test/test_compiler_assemble.py | 2 +- Lib/test/test_compiler_codegen.py | 2 +- Lib/test/test_dis.py | 56 +- ...-08-05-09-06-56.gh-issue-105848.Drc-1-.rst | 3 + Objects/exception_handling_notes.txt | 2 +- Programs/test_frozenmain.h | 22 +- Python/bytecodes.c | 372 ++++++------ Python/compile.c | 19 +- Python/executor_cases.c.h | 231 ++++---- Python/flowgraph.c | 8 +- Python/generated_cases.c.h | 555 +++++++++--------- Python/specialize.c | 8 + 16 files changed, 627 insertions(+), 682 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 2217c3a8035459..0b44d160de58a7 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -56,7 +56,7 @@ the following command can be used to display the disassembly of >>> dis.dis(myfunc) 2 0 RESUME 0 - 3 2 LOAD_GLOBAL 1 (NULL + len) + 3 2 LOAD_GLOBAL 1 (len + NULL) 12 LOAD_FAST 0 (alist) 14 CALL 1 22 RETURN_VALUE diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 02303c42c75c8a..9f4437c09e92cb 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -273,11 +273,11 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case BUILD_CONST_KEY_MAP: return oparg + 1; case DICT_UPDATE: - return 1; + return (oparg - 1) + 2; case DICT_MERGE: - return 1; + return (oparg - 1) + 5; case MAP_ADD: - return 2; + return (oparg - 1) + 3; case INSTRUMENTED_LOAD_SUPER_ATTR: return 3; case LOAD_SUPER_ATTR: @@ -719,11 +719,11 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case BUILD_CONST_KEY_MAP: return 1; case DICT_UPDATE: - return 0; + return (oparg - 1) + 1; case DICT_MERGE: - return 0; + return (oparg - 1) + 4; case MAP_ADD: - return 0; + return (oparg - 1) + 1; case INSTRUMENTED_LOAD_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_SUPER_ATTR: @@ -735,7 +735,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_ZERO_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_SUPER_ATTR_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; + return 1; case LOAD_SUPER_ATTR_METHOD: return 2; case LOAD_ATTR: @@ -753,9 +753,9 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_ATTR_CLASS: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_PROPERTY: - return ((oparg & 1) ? 1 : 0) + 1; + return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return ((oparg & 1) ? 1 : 0) + 1; + return 1; case STORE_ATTR_INSTANCE_VALUE: return 0; case STORE_ATTR_WITH_HINT: diff --git a/Lib/dis.py b/Lib/dis.py index b167773dfe7b0c..bf1a1e2ff7ac19 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -552,15 +552,15 @@ def _get_instructions_bytes(code, varname_from_oparg=None, if deop == LOAD_GLOBAL: argval, argrepr = _get_name_info(arg//2, get_name) if (arg & 1) and argrepr: - argrepr = "NULL + " + argrepr + argrepr = f"{argrepr} + NULL" elif deop == LOAD_ATTR: argval, argrepr = _get_name_info(arg//2, get_name) if (arg & 1) and argrepr: - argrepr = "NULL|self + " + argrepr + argrepr = f"{argrepr} + NULL|self" elif deop == LOAD_SUPER_ATTR: argval, argrepr = _get_name_info(arg//4, get_name) if (arg & 1) and argrepr: - argrepr = "NULL|self + " + argrepr + argrepr = f"{argrepr} + NULL|self" else: argval, argrepr = _get_name_info(arg, get_name) elif deop in hasjabs: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 16a82bef2ba71f..5f0d659b1ed535 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -453,6 +453,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) # Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) # Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) +# Python 3.13a1 3558 (Reorder the stack items for CALL) # Python 3.14 will start with 3600 @@ -469,7 +470,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3557).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3558).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 6df72cbc54666b..5696433e529d0a 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -94,12 +94,12 @@ def inner(): instructions = [ ('RESUME', 0,), - ('PUSH_NULL', 0, 1), ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), ('MAKE_FUNCTION', 0, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), + ('PUSH_NULL', 0, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index d99bb8c6cd472d..6d7731ddba02c5 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -41,8 +41,8 @@ def test_for_loop(self): loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), ('STORE_NAME', 1, 1), - ('PUSH_NULL', None, 2), ('LOAD_NAME', 2, 2), + ('PUSH_NULL', None, 2), ('LOAD_NAME', 1, 2), ('CALL', 1, 2), ('POP_TOP', None), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 642b26163ab248..f49c60a01a54ea 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -102,7 +102,7 @@ def _f(a): dis_f = """\ %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + print) +%3d LOAD_GLOBAL 1 (print + NULL) LOAD_FAST 0 (a) CALL 1 POP_TOP @@ -131,7 +131,7 @@ def bug708901(): dis_bug708901 = """\ %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + range) +%3d LOAD_GLOBAL 1 (range + NULL) LOAD_CONST 1 (1) %3d LOAD_CONST 2 (10) @@ -236,7 +236,7 @@ def wrap_func_w_kwargs(): dis_kw_names = """\ %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + func_w_kwargs) +%3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) LOAD_CONST 1 (1) LOAD_CONST 2 (2) LOAD_CONST 3 (5) @@ -345,8 +345,8 @@ def wrap_func_w_kwargs(): LOAD_CONST 1 ('x') STORE_SUBSCR - 3 PUSH_NULL - LOAD_NAME 3 (fun) + 3 LOAD_NAME 3 (fun) + PUSH_NULL LOAD_CONST 0 (1) CALL 1 LOAD_NAME 2 (__annotations__) @@ -355,8 +355,8 @@ def wrap_func_w_kwargs(): 4 LOAD_CONST 0 (1) LOAD_NAME 4 (lst) - PUSH_NULL LOAD_NAME 3 (fun) + PUSH_NULL LOAD_CONST 3 (0) CALL 1 STORE_SUBSCR @@ -615,14 +615,14 @@ def _tryfinallyconst(b): %3d LOAD_FAST 0 (a) -%3d PUSH_NULL - LOAD_FAST 1 (b) +%3d LOAD_FAST 1 (b) + PUSH_NULL CALL 0 POP_TOP RETURN_VALUE >> PUSH_EXC_INFO - PUSH_NULL LOAD_FAST 1 (b) + PUSH_NULL CALL 0 POP_TOP RERAISE 0 @@ -644,14 +644,14 @@ def _tryfinallyconst(b): %3d NOP -%3d PUSH_NULL - LOAD_FAST 0 (b) +%3d LOAD_FAST 0 (b) + PUSH_NULL CALL 0 POP_TOP RETURN_CONST 1 (1) PUSH_EXC_INFO - PUSH_NULL LOAD_FAST 0 (b) + PUSH_NULL CALL 0 POP_TOP RERAISE 0 @@ -710,7 +710,7 @@ def foo(x): %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + list) +%3d LOAD_GLOBAL 1 (list + NULL) LOAD_FAST 0 (x) BUILD_TUPLE 1 LOAD_CONST 1 ( at 0x..., file "%s", line %d>) @@ -792,7 +792,7 @@ def loop_test(): >> FOR_ITER_LIST 14 (to 48) STORE_FAST 0 (i) -%3d LOAD_GLOBAL_MODULE 1 (NULL + load_test) +%3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) CALL_PY_WITH_DEFAULTS 1 POP_TOP @@ -1230,8 +1230,8 @@ def test_call_specialize(self): call_quicken = """\ 0 RESUME 0 - 1 PUSH_NULL - LOAD_NAME 0 (str) + 1 LOAD_NAME 0 (str) + PUSH_NULL LOAD_CONST 0 (1) CALL_NO_KW_STR_1 1 RETURN_VALUE @@ -1629,7 +1629,7 @@ def _prepare_test_cases(): Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), @@ -1659,7 +1659,7 @@ def _prepare_test_cases(): Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), @@ -1673,7 +1673,7 @@ def _prepare_test_cases(): expected_opinfo_inner = [ Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), @@ -1686,13 +1686,13 @@ def _prepare_test_cases(): expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='NULL + range', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), @@ -1709,14 +1709,14 @@ def _prepare_test_cases(): Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), Instruction(opname='JUMP_FORWARD', opcode=110, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), @@ -1738,7 +1738,7 @@ def _prepare_test_cases(): Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_BACKWARD', opcode=140, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), @@ -1750,7 +1750,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), @@ -1759,7 +1759,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), @@ -1782,7 +1782,7 @@ def _prepare_test_cases(): Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), @@ -1793,7 +1793,7 @@ def _prepare_test_cases(): Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst new file mode 100644 index 00000000000000..6c1c3229475f6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst @@ -0,0 +1,3 @@ +Modify the bytecode so that the actual callable for a :opcode:`CALL` is at a +consistent position on the stack (regardless of whether or not +bound-method-calling optimizations are active). diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt index 7de01fdbf5ff48..387ef935ce739e 100644 --- a/Objects/exception_handling_notes.txt +++ b/Objects/exception_handling_notes.txt @@ -47,7 +47,7 @@ a table to determine where to jump to when an exception is raised. 2 2 NOP - 3 4 LOAD_GLOBAL 1 (NULL + g) + 3 4 LOAD_GLOBAL 1 (g + NULL) 16 LOAD_CONST 1 (0) 18 PRECALL 1 22 CALL 1 diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 9058327e846dc3..0dca507e28bc14 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -2,14 +2,14 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,164,0,0,0,151,0,100,0,100,1, - 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,0,0,0,0,0,0,1,0,2,0,101,2, + 108,0,90,0,100,0,100,1,108,1,90,1,101,2,2,0, + 100,2,171,1,0,0,0,0,0,0,1,0,101,2,2,0, 100,3,101,0,106,6,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,171,2,0,0,0,0,0,0, - 1,0,2,0,101,1,106,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,0,0,0,0,0, + 1,0,101,1,106,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,2,0,171,0,0,0,0,0, 0,0,100,4,25,0,0,0,90,5,100,5,68,0,93,20, - 0,0,90,6,2,0,101,2,100,6,101,6,40,0,100,7, + 0,0,90,6,101,2,2,0,100,6,101,6,40,0,100,7, 101,5,101,6,25,0,0,0,40,0,157,4,171,1,0,0, 0,0,0,0,1,0,140,22,0,0,4,0,121,1,41,8, 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, @@ -27,12 +27,12 @@ unsigned char M_test_frozenmain[] = { 0,0,218,3,107,101,121,169,0,243,0,0,0,0,250,18, 116,101,115,116,95,102,114,111,122,101,110,109,97,105,110,46, 112,121,250,8,60,109,111,100,117,108,101,62,114,18,0,0, - 0,1,0,0,0,115,102,0,0,0,240,3,1,1,1,243, + 0,1,0,0,0,115,99,0,0,0,240,3,1,1,1,243, 8,0,1,11,219,0,24,225,0,5,208,6,26,212,0,27, 217,0,5,128,106,144,35,151,40,145,40,212,0,27,216,9, - 38,208,9,26,215,9,38,209,9,38,211,9,40,168,24,209, - 9,50,128,6,240,2,6,12,2,242,0,7,1,42,128,67, - 241,14,0,5,10,136,71,144,67,144,53,152,2,152,54,160, - 35,153,59,152,45,208,10,40,214,4,41,241,15,7,1,42, - 114,16,0,0,0, + 26,215,9,38,210,9,38,211,9,40,168,24,209,9,50,128, + 6,240,2,6,12,2,242,0,7,1,42,128,67,241,14,0, + 5,10,136,71,144,67,144,53,152,2,152,54,160,35,153,59, + 152,45,208,10,40,214,4,41,241,15,7,1,42,114,16,0, + 0,0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d6bfb624c77133..b2281abc6663da 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1315,7 +1315,7 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- null if (oparg & 1), v)) { + inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1331,10 +1331,10 @@ dummy_func( if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1343,17 +1343,17 @@ dummy_func( } ERROR_IF(true, error); } - Py_INCREF(v); + Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); + if (res == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); + if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1378,7 +1378,7 @@ dummy_func( assert(DK_IS_UNICODE(dict->ma_keys)); } - op(_LOAD_GLOBAL_MODULE, (index/1 -- null if (oparg & 1), res)) { + op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -1388,7 +1388,7 @@ dummy_func( null = NULL; } - op(_LOAD_GLOBAL_BUILTINS, (index/1 -- null if (oparg & 1), res)) { + op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -1614,8 +1614,7 @@ dummy_func( ERROR_IF(map == NULL, error); } - inst(DICT_UPDATE, (update --)) { - PyObject *dict = PEEK(oparg + 1); // update is still on the stack + inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -1628,26 +1627,23 @@ dummy_func( DECREF_INPUTS(); } - inst(DICT_MERGE, (update --)) { - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - + inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, callable, update); DECREF_INPUTS(); ERROR_IF(true, error); } DECREF_INPUTS(); } - inst(MAP_ADD, (key, value --)) { - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + inst(MAP_ADD, (dict, unused[oparg - 1], key, value -- dict, unused[oparg - 1])) { assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); } - inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused if (oparg & 1), unused)) { + inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) { _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions @@ -1660,7 +1656,7 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- res2 if (oparg & 1), res)) { + inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -1704,9 +1700,10 @@ dummy_func( } DECREF_INPUTS(); ERROR_IF(super == NULL, error); - res = PyObject_GetAttr(super, name); + attr = PyObject_GetAttr(super, name); Py_DECREF(super); - ERROR_IF(res == NULL, error); + ERROR_IF(attr == NULL, error); + null = NULL; } pseudo(LOAD_SUPER_METHOD) = { @@ -1721,18 +1718,18 @@ dummy_func( LOAD_SUPER_ATTR, }; - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- res2 if (oparg & 1), res)) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(attr == NULL, error); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- res2, res)) { + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1740,20 +1737,19 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); - if (res2 == NULL) { + if (attr == NULL) { Py_DECREF(self); ERROR_IF(true, error); } if (method_found) { - res = self; // transfer ownership + self_or_null = self; // transfer ownership } else { Py_DECREF(self); - res = res2; - res2 = NULL; + self_or_null = NULL; } } @@ -1772,7 +1768,7 @@ dummy_func( LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; - inst(LOAD_ATTR, (unused/9, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1787,16 +1783,15 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -1807,16 +1802,15 @@ dummy_func( NULL | meth | arg1 | ... | argN */ DECREF_INPUTS(); - ERROR_IF(meth == NULL, error); - res2 = NULL; - res = meth; + ERROR_IF(attr == NULL, error); + self_or_null = NULL; } } else { /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); + attr = PyObject_GetAttr(owner, name); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(attr == NULL, error); } } @@ -1837,13 +1831,13 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- res2 if (oparg & 1), res)) { + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } @@ -1854,7 +1848,7 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -1862,15 +1856,15 @@ dummy_func( assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } - inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -1886,49 +1880,50 @@ dummy_func( if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } - DEOPT_IF(res == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } - inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); char *addr = (char *)owner + index; - res = *(PyObject **)addr; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } - inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); - res2 = NULL; - res = descr; - assert(res != NULL); - Py_INCREF(res); + null = NULL; + attr = descr; + assert(attr != NULL); + Py_INCREF(attr); DECREF_INPUTS(); } - inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { + inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -1945,16 +1940,15 @@ dummy_func( Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); } - inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused if (oparg & 1), unused)) { + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -1972,9 +1966,7 @@ dummy_func( Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); @@ -2728,80 +2720,80 @@ dummy_func( exc_info->exc_value = Py_NewRef(new_exc); } - inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (1), res)) { + inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); /* Cached method object */ - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - res2 = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res = self; + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; } - inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (1), res)) { + inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; } - inst(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (0), res)) { + inst(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - res = Py_NewRef(descr); + attr = Py_NewRef(descr); } - inst(LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (0), res)) { + inst(LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - res = Py_NewRef(descr); + attr = Py_NewRef(descr); } - inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (1), res)) { + inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - Py_ssize_t dictoffset = self_cls->tp_dictoffset; + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)self + dictoffset); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; } inst(KW_NAMES, (--)) { @@ -2811,9 +2803,9 @@ dummy_func( } inst(INSTRUMENTED_CALL, ( -- )) { - int is_meth = PEEK(oparg+2) != NULL; + int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(total_args + 1); + PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( @@ -2855,11 +2847,9 @@ dummy_func( // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.) // On exit, the stack is [result]. // When calling Python, inline the call using DISPATCH_INLINED(). - inst(CALL, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - int is_meth = method != NULL; + inst(CALL, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2873,13 +2863,12 @@ dummy_func( STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { - is_meth = 1; // For consistenct; it's dead, though + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable)->im_self; args[0] = Py_NewRef(self); - method = ((PyMethodObject *)callable)->im_func; + PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); Py_DECREF(callable); callable = method; @@ -2915,7 +2904,7 @@ dummy_func( kwnames); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -2943,25 +2932,23 @@ dummy_func( // Start out with [NULL, bound_method, arg1, arg2, ...] // Transform to [callable, self, arg1, arg2, ...] // Then fall through to CALL_PY_EXACT_ARGS - inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, method, callable, unused[oparg] -- unused)) { - DEOPT_IF(method != NULL, CALL); + inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, callable, null, unused[oparg] -- unused)) { + DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // callable + PEEK(oparg + 1) = Py_NewRef(self); // self_or_null PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // method + PEEK(oparg + 2) = Py_NewRef(meth); // callable Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { + inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -2983,13 +2970,11 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -3021,7 +3006,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3033,7 +3018,7 @@ dummy_func( Py_DECREF(&PyType_Type); // I.e., callable } - inst(CALL_NO_KW_STR_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3047,7 +3032,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3061,7 +3046,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, null, callable, args[oparg] -- unused)) { + inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -3124,11 +3109,9 @@ dummy_func( } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - int is_meth = method != NULL; + inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3149,13 +3132,11 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3180,13 +3161,11 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3215,12 +3194,10 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3250,13 +3227,11 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_LEN, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); /* len(o) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3277,13 +3252,11 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3307,19 +3280,19 @@ dummy_func( } // This is secretly a super-instruction - inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, method, self, args[oparg] -- unused)) { + inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); - assert(method != NULL); + assert(self != NULL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); - Py_DECREF(method); + Py_DECREF(callable); STACK_SHRINK(3); // CALL + POP_TOP SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); @@ -3327,23 +3300,21 @@ dummy_func( DISPATCH(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3361,19 +3332,17 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { - int is_meth = method != NULL; + inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = callable->d_common.d_type; + PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); @@ -3393,21 +3362,20 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3425,22 +3393,20 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3460,7 +3426,7 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(CALL_FUNCTION_EX, (unused, func, callargs, kwargs if (oparg & 1) -- result)) { + inst(CALL_FUNCTION_EX, (func, unused, callargs, kwargs if (oparg & 1) -- result)) { // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -3523,7 +3489,7 @@ dummy_func( result = PyObject_Call(func, callargs, kwargs); } DECREF_INPUTS(); - assert(PEEK(3 + (oparg & 1)) == NULL); + assert(PEEK(2 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/compile.c b/Python/compile.c index b673e3ac6c1cc5..68af5d61ac8de0 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2361,11 +2361,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) int is_generic = asdl_seq_LEN(type_params) > 0; - if (is_generic) { - // Used by the CALL to the type parameters function. - ADDOP(c, loc, PUSH_NULL); - } - funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { return ERROR; @@ -2436,8 +2431,12 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) Py_DECREF(co); if (num_typeparam_args > 0) { ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); + ADDOP_I(c, loc, CALL, num_typeparam_args - 1); + } + else { + ADDOP(c, loc, PUSH_NULL); + ADDOP_I(c, loc, CALL, 0); } - ADDOP_I(c, loc, CALL, num_typeparam_args); } RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); @@ -2565,8 +2564,8 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) // these instructions should be attributed to the class line, // not a decorator line loc = LOC(s); - ADDOP(c, loc, PUSH_NULL); ADDOP(c, loc, LOAD_BUILD_CLASS); + ADDOP(c, loc, PUSH_NULL); /* 3. load a function (or closure) made from the code object */ if (compiler_make_closure(c, loc, co, 0) < 0) { @@ -2598,7 +2597,6 @@ compiler_class(struct compiler *c, stmt_ty s) int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); - ADDOP(c, loc, PUSH_NULL); PyObject *type_params_name = PyUnicode_FromFormat("", s->v.ClassDef.name); if (!type_params_name) { @@ -2666,6 +2664,7 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } Py_DECREF(co); + ADDOP(c, loc, PUSH_NULL); ADDOP_I(c, loc, CALL, 0); } else { RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, @@ -2716,7 +2715,6 @@ compiler_typealias(struct compiler *c, stmt_ty s) int is_generic = asdl_seq_LEN(type_params) > 0; PyObject *name = s->v.TypeAlias.name->v.Name.id; if (is_generic) { - ADDOP(c, loc, PUSH_NULL); PyObject *type_params_name = PyUnicode_FromFormat("", name); if (!type_params_name) { @@ -2756,6 +2754,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) return ERROR; } Py_DECREF(co); + ADDOP(c, loc, PUSH_NULL); ADDOP_I(c, loc, CALL, 0); } RETURN_IF_ERROR(compiler_nameop(c, loc, name, Store)); @@ -4994,9 +4993,9 @@ compiler_call(struct compiler *c, expr_ty e) return SUCCESS; } RETURN_IF_ERROR(check_caller(c, e->v.Call.func)); + VISIT(c, expr, e->v.Call.func); location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); - VISIT(c, expr, e->v.Call.func); loc = LOC(e); return compiler_call_helper(c, loc, 0, e->v.Call.args, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 27be8a383989ee..d6d541a3b61ab4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1062,8 +1062,8 @@ case LOAD_GLOBAL: { static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; PyObject *null = NULL; - PyObject *v; #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1079,10 +1079,10 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1091,17 +1091,17 @@ } if (true) goto error; } - Py_INCREF(v); + Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1112,8 +1112,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = v; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } @@ -1136,8 +1136,8 @@ } case _LOAD_GLOBAL_MODULE: { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; uint16_t index = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -1148,14 +1148,14 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } case _LOAD_GLOBAL_BUILTINS: { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; uint16_t index = (uint16_t)operand; PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); @@ -1166,8 +1166,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } @@ -1445,8 +1445,9 @@ case DICT_UPDATE: { PyObject *update; + PyObject *dict; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack + dict = stack_pointer[-2 - (oparg - 1)]; if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -1463,11 +1464,13 @@ case DICT_MERGE: { PyObject *update; + PyObject *dict; + PyObject *callable; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; } @@ -1479,9 +1482,10 @@ case MAP_ADD: { PyObject *value; PyObject *key; + PyObject *dict; value = stack_pointer[-1]; key = stack_pointer[-2]; - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + dict = stack_pointer[-3 - (oparg - 1)]; assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references @@ -1494,8 +1498,7 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -1504,15 +1507,13 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (res == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (0 ? 1 : 0)] = attr; break; } @@ -1520,8 +1521,8 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2; - PyObject *res; + PyObject *attr; + PyObject *self_or_null; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -1532,32 +1533,31 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); - if (res2 == NULL) { + if (attr == NULL) { Py_DECREF(self); if (true) goto pop_3_error; } if (method_found) { - res = self; // transfer ownership + self_or_null = self; // transfer ownership } else { Py_DECREF(self); - res = res2; - res2 = NULL; + self_or_null = NULL; } STACK_SHRINK(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self_or_null; break; } case LOAD_ATTR: { static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1573,16 +1573,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -1593,20 +1592,19 @@ NULL | meth | arg1 | ... | argN */ Py_DECREF(owner); - if (meth == NULL) goto pop_1_error; - res2 = NULL; - res = meth; + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; } } else { /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); + attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (res == NULL) goto pop_1_error; + if (attr == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } break; } @@ -1632,20 +1630,20 @@ case _LOAD_ATTR_INSTANCE_VALUE: { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } @@ -2157,12 +2155,12 @@ case CALL_NO_KW_TYPE_1: { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2180,12 +2178,12 @@ case CALL_NO_KW_STR_1: { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2205,12 +2203,12 @@ case CALL_NO_KW_TUPLE_1: { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2244,18 +2242,16 @@ case CALL_NO_KW_BUILTIN_O: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2286,18 +2282,16 @@ case CALL_NO_KW_BUILTIN_FAST: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2332,18 +2326,16 @@ case CALL_NO_KW_LEN: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2370,18 +2362,16 @@ case CALL_NO_KW_ISINSTANCE: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2410,26 +2400,26 @@ case CALL_NO_KW_METHOD_DESCRIPTOR_O: { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -2453,24 +2443,25 @@ case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -2494,25 +2485,25 @@ case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index a72b85d45aa27d..5b6b3f3cfefb03 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1536,10 +1536,10 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) break; case KW_NAMES: break; - case PUSH_NULL: - if (nextop == LOAD_GLOBAL && (inst[1].i_opcode & 1) == 0) { - INSTR_SET_OP0(inst, NOP); - inst[1].i_oparg |= 1; + case LOAD_GLOBAL: + if (nextop == PUSH_NULL && (oparg & 1) == 0) { + INSTR_SET_OP1(inst, LOAD_GLOBAL, oparg | 1); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); } break; case COMPARE_OP: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d7db8b07005fd1..cf20b869b8182f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1695,8 +1695,8 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; PyObject *null = NULL; - PyObject *v; #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1712,10 +1712,10 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1724,17 +1724,17 @@ } if (true) goto error; } - Py_INCREF(v); + Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1745,15 +1745,15 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = v; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); @@ -1775,15 +1775,15 @@ } STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); @@ -1813,8 +1813,8 @@ } STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 4; DISPATCH(); } @@ -2105,8 +2105,9 @@ TARGET(DICT_UPDATE) { PyObject *update; + PyObject *dict; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack + dict = stack_pointer[-2 - (oparg - 1)]; if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2123,11 +2124,13 @@ TARGET(DICT_MERGE) { PyObject *update; + PyObject *dict; + PyObject *callable; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; } @@ -2139,9 +2142,10 @@ TARGET(MAP_ADD) { PyObject *value; PyObject *key; + PyObject *dict; value = stack_pointer[-1]; key = stack_pointer[-2]; - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + dict = stack_pointer[-3 - (oparg - 1)]; assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references @@ -2166,8 +2170,8 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -2216,13 +2220,14 @@ Py_DECREF(class); Py_DECREF(self); if (super == NULL) goto pop_3_error; - res = PyObject_GetAttr(super, name); + attr = PyObject_GetAttr(super, name); Py_DECREF(super); - if (res == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error; + null = NULL; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 1; DISPATCH(); } @@ -2231,8 +2236,7 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -2241,15 +2245,13 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (res == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (0 ? 1 : 0)] = attr; next_instr += 1; DISPATCH(); } @@ -2258,8 +2260,8 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2; - PyObject *res; + PyObject *attr; + PyObject *self_or_null; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -2270,24 +2272,23 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); - if (res2 == NULL) { + if (attr == NULL) { Py_DECREF(self); if (true) goto pop_3_error; } if (method_found) { - res = self; // transfer ownership + self_or_null = self; // transfer ownership } else { Py_DECREF(self); - res = res2; - res2 = NULL; + self_or_null = NULL; } STACK_SHRINK(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self_or_null; next_instr += 1; DISPATCH(); } @@ -2296,8 +2297,8 @@ PREDICTED(LOAD_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2313,16 +2314,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -2333,28 +2333,27 @@ NULL | meth | arg1 | ... | argN */ Py_DECREF(owner); - if (meth == NULL) goto pop_1_error; - res2 = NULL; - res = meth; + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; } } else { /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); + attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (res == NULL) goto pop_1_error; + if (attr == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -2374,24 +2373,24 @@ { uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); } STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); @@ -2402,23 +2401,23 @@ assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); @@ -2437,29 +2436,29 @@ if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } - DEOPT_IF(res == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); @@ -2467,41 +2466,41 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); char *addr = (char *)owner + index; - res = *(PyObject **)addr; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *cls; - PyObject *res2 = NULL; - PyObject *res; - cls = stack_pointer[-1]; + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); - res2 = NULL; - res = descr; - assert(res != NULL); - Py_INCREF(res); - Py_DECREF(cls); + null = NULL; + attr = descr; + assert(attr != NULL); + Py_INCREF(attr); + Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } @@ -2512,6 +2511,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2528,14 +2528,11 @@ Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2544,6 +2541,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2561,15 +2559,12 @@ Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -3499,128 +3494,128 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + PyObject *owner; + PyObject *attr; PyObject *self; - PyObject *res2; - PyObject *res; - self = stack_pointer[-1]; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); /* Cached method object */ - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - res2 = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res = self; + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; STACK_GROW(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { + PyObject *owner; + PyObject *attr; PyObject *self; - PyObject *res2; - PyObject *res; - self = stack_pointer[-1]; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; STACK_GROW(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PyObject *self; - PyObject *res; - self = stack_pointer[-1]; + PyObject *owner; + PyObject *attr; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(self); - res = Py_NewRef(descr); - stack_pointer[-1] = res; + Py_DECREF(owner); + attr = Py_NewRef(descr); + stack_pointer[-1 - (0 ? 1 : 0)] = attr; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PyObject *self; - PyObject *res; - self = stack_pointer[-1]; + PyObject *owner; + PyObject *attr; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(self); - res = Py_NewRef(descr); - stack_pointer[-1] = res; + Py_DECREF(owner); + attr = Py_NewRef(descr); + stack_pointer[-1 - (0 ? 1 : 0)] = attr; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + PyObject *owner; + PyObject *attr; PyObject *self; - PyObject *res2; - PyObject *res; - self = stack_pointer[-1]; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - Py_ssize_t dictoffset = self_cls->tp_dictoffset; + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)self + dictoffset); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; STACK_GROW(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; next_instr += 9; DISPATCH(); } @@ -3633,9 +3628,9 @@ } TARGET(INSTRUMENTED_CALL) { - int is_meth = PEEK(oparg+2) != NULL; + int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(total_args + 1); + PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( @@ -3651,16 +3646,14 @@ PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; - int is_meth = method != NULL; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3674,13 +3667,12 @@ STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { - is_meth = 1; // For consistenct; it's dead, though + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable)->im_self; args[0] = Py_NewRef(self); - method = ((PyMethodObject *)callable)->im_func; + PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); Py_DECREF(callable); callable = method; @@ -3716,7 +3708,7 @@ kwnames); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3747,17 +3739,17 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + PyObject *null; PyObject *callable; - PyObject *method; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; - DEOPT_IF(method != NULL, CALL); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // callable + PEEK(oparg + 1) = Py_NewRef(self); // self_or_null PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // method + PEEK(oparg + 2) = Py_NewRef(meth); // callable Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); STACK_SHRINK(oparg); @@ -3767,18 +3759,16 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -3804,18 +3794,16 @@ TARGET(CALL_PY_WITH_DEFAULTS) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -3851,12 +3839,12 @@ TARGET(CALL_NO_KW_TYPE_1) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3875,12 +3863,12 @@ TARGET(CALL_NO_KW_STR_1) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3901,12 +3889,12 @@ TARGET(CALL_NO_KW_TUPLE_1) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3927,11 +3915,11 @@ TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -4002,16 +3990,14 @@ TARGET(CALL_BUILTIN_CLASS) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; - int is_meth = method != NULL; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4039,18 +4025,16 @@ TARGET(CALL_NO_KW_BUILTIN_O) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4082,18 +4066,16 @@ TARGET(CALL_NO_KW_BUILTIN_FAST) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4129,17 +4111,15 @@ TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4176,18 +4156,16 @@ TARGET(CALL_NO_KW_LEN) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4215,18 +4193,16 @@ TARGET(CALL_NO_KW_ISINSTANCE) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4257,22 +4233,22 @@ TARGET(CALL_NO_KW_LIST_APPEND) { PyObject **args; PyObject *self; - PyObject *method; + PyObject *callable; args = stack_pointer - oparg; self = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); - assert(method != NULL); + assert(self != NULL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); - Py_DECREF(method); + Py_DECREF(callable); STACK_SHRINK(3); // CALL + POP_TOP SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); @@ -4284,26 +4260,26 @@ TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -4328,22 +4304,22 @@ TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; - int is_meth = method != NULL; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = callable->d_common.d_type; + PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); @@ -4370,24 +4346,25 @@ TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -4412,25 +4389,25 @@ TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -4463,7 +4440,7 @@ PyObject *result; if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-2 - (oparg & 1 ? 1 : 0)]; + func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4528,7 +4505,7 @@ Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - assert(PEEK(3 + (oparg & 1)) == NULL); + assert(PEEK(2 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); diff --git a/Python/specialize.c b/Python/specialize.c index 855252e066dea1..98455ae3efc60d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -793,6 +793,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) if (!function_check_args(fget, 1, LOAD_ATTR)) { goto fail; } + if (instr->op.arg & 1) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + goto fail; + } uint32_t version = function_get_version(fget, LOAD_ATTR); if (version == 0) { goto fail; @@ -859,6 +863,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) if (!function_check_args(descr, 2, LOAD_ATTR)) { goto fail; } + if (instr->op.arg & 1) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + goto fail; + } uint32_t version = function_get_version(descr, LOAD_ATTR); if (version == 0) { goto fail; From 326f0ba1c5dda1d9613dbba11ea2470654b0d9c8 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 9 Aug 2023 12:14:50 -0700 Subject: [PATCH 091/598] GH-106485: Dematerialize instance dictionaries when possible (GH-106539) --- Include/internal/pycore_dict.h | 4 +++ Include/pystats.h | 1 + Lib/test/test_opcache.py | 8 +++-- ...-07-16-07-55-19.gh-issue-106485.wPb1bH.rst | 2 ++ Objects/dictobject.c | 33 +++++++++++++++++++ Objects/typeobject.c | 3 -- Python/bytecodes.c | 18 ++++++---- Python/executor_cases.c.h | 6 ++-- Python/generated_cases.c.h | 18 ++++++---- Python/specialize.c | 19 ++++++++--- 10 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 26a913435ef9cf..6ae81c033c43aa 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -10,6 +10,7 @@ extern "C" { #endif #include "pycore_dict_state.h" +#include "pycore_object.h" #include "pycore_runtime.h" // _PyRuntime // Unsafe flavor of PyDict_GetItemWithError(): no error checking @@ -62,6 +63,8 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); +extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); + /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. */ @@ -196,6 +199,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); +extern int _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); extern PyObject *_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, diff --git a/Include/pystats.h b/Include/pystats.h index e24aef5fe8072b..b1957596745f00 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -65,6 +65,7 @@ typedef struct _object_stats { uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; + uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 564dc4745ae64e..7d317c012a304e 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -865,8 +865,10 @@ class C: items = [] for _ in range(self.ITEMS): item = C() - item.__dict__ item.a = None + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) items.append(item) return items @@ -932,7 +934,9 @@ class C: items = [] for _ in range(self.ITEMS): item = C() - item.__dict__ + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) items.append(item) return items diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst new file mode 100644 index 00000000000000..1f80082821edac --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst @@ -0,0 +1,2 @@ +Reduce the number of materialized instances dictionaries by dematerializing +them when possible. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 41ae1fca90b468..931103cdc1a064 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5464,6 +5464,35 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } +// Return 1 if the dict was dematerialized, 0 otherwise. +int +_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) +{ + assert(_PyObject_DictOrValuesPointer(obj) == dorv); + assert(!_PyDictOrValues_IsValues(*dorv)); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); + if (dict == NULL) { + return 0; + } + // It's likely that this dict still shares its keys (if it was materialized + // on request and not heavily modified): + assert(PyDict_CheckExact(dict)); + assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); + if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { + return 0; + } + assert(dict->ma_values); + // We have an opportunity to do something *really* cool: dematerialize it! + _PyDictKeys_DecRef(dict->ma_keys); + _PyDictOrValues_SetValues(dorv, dict->ma_values); + OBJECT_STAT_INC(dict_dematerialized); + // Don't try this at home, kids: + dict->ma_keys = NULL; + dict->ma_values = NULL; + Py_DECREF(dict); + return 1; +} + int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) @@ -5688,6 +5717,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = _PyDictOrValues_GetDict(*dorv_ptr); if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); + OBJECT_STAT_INC(dict_materialized_on_request); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); dorv_ptr->dict = dict; } @@ -5731,6 +5761,9 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = *dictptr; if (dict == NULL) { dictkeys_incref(cached); + if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { + OBJECT_STAT_INC(dict_materialized_on_request); + } dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 76809494dd8817..71e96f5af4e419 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4965,9 +4965,6 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) return res; } -extern void -_PyDictKeys_DecRef(PyDictKeysObject *keys); - static void type_dealloc_common(PyTypeObject *type) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b2281abc6663da..5efa36fcf5c629 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1827,8 +1827,10 @@ dummy_func( op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); } op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { @@ -2727,8 +2729,10 @@ dummy_func( assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); @@ -2757,8 +2761,10 @@ dummy_func( assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d6d541a3b61ab4..a7b5054417ed8f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1623,8 +1623,10 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cf20b869b8182f..ccf43c727b9e0f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2366,8 +2366,10 @@ { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -3507,8 +3509,10 @@ assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); @@ -3559,8 +3563,10 @@ assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); diff --git a/Python/specialize.c b/Python/specialize.c index 98455ae3efc60d..2d514c0dc476d3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -192,6 +192,7 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); @@ -685,8 +686,10 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + if (_PyDictOrValues_IsValues(*dorv) || + _PyObject_MakeInstanceAttributesFromDict(owner, dorv)) + { // Virtual dictionary PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); @@ -704,12 +707,16 @@ specialize_dict_access( instr->op.code = values_op; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } // We found an instance with a __dict__. + if (dict->ma_values) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); + return 0; + } Py_ssize_t index = _PyDict_LookupIndex(dict, name); if (index != (uint16_t)index) { @@ -1100,9 +1107,11 @@ PyObject *descr, DescriptorClassification kind, bool is_method) assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - if (!_PyDictOrValues_IsValues(dorv)) { + if (!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) + { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); return 0; } From 1e229e2c3d212accbd5fbe3a46cd42f8252b2868 Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Thu, 10 Aug 2023 00:42:16 +0200 Subject: [PATCH 092/598] gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) --- .../Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst | 1 + PCbuild/find_python.bat | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst diff --git a/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst new file mode 100644 index 00000000000000..d3723353470ce2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst @@ -0,0 +1 @@ +When calling ``find_python.bat`` with ``-q`` it did not properly silence the output of nuget. That is now fixed. diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index 7af5503d80a0fc..d3f62c93869003 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -52,7 +52,7 @@ @if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) @if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) @if NOT exist "%_Py_NUGET%" ( - @echo Downloading nuget... + @if not "%_Py_Quiet%"=="1" @echo Downloading nuget... @rem NB: Must use single quotes around NUGET here, NOT double! @rem Otherwise, a space in the path would break things @rem If it fails, retry with any available copy of Python @@ -63,7 +63,11 @@ ) @if not "%_Py_Quiet%"=="1" @echo Installing Python via nuget... -@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +@if not "%_Py_Quiet%"=="1" ( + @"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) else ( + @"%_Py_NUGET%" install pythonx86 -Verbosity quiet -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) @rem Quote it here; it's not quoted later because "py -x.y" wouldn't work @if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found From 2ec16fed14aae896e38dd5bd9e73e2eddc974439 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 9 Aug 2023 16:42:32 -0600 Subject: [PATCH 093/598] gh-91054: make code watcher tests resilient to other watchers (#107821) --- Modules/_testcapi/watchers.c | 14 +++++++++++--- Tools/c-analyzer/cpython/ignored.tsv | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 4cf567b3314980..8a264bba4ed6ed 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -295,6 +295,7 @@ _testcapi_unwatch_type_impl(PyObject *module, int watcher_id, PyObject *type) // Test code object watching #define NUM_CODE_WATCHERS 2 +static int code_watcher_ids[NUM_CODE_WATCHERS] = {-1, -1}; static int num_code_object_created_events[NUM_CODE_WATCHERS] = {0, 0}; static int num_code_object_destroyed_events[NUM_CODE_WATCHERS] = {0, 0}; @@ -345,11 +346,13 @@ add_code_watcher(PyObject *self, PyObject *which_watcher) long which_l = PyLong_AsLong(which_watcher); if (which_l == 0) { watcher_id = PyCode_AddWatcher(first_code_object_callback); + code_watcher_ids[0] = watcher_id; num_code_object_created_events[0] = 0; num_code_object_destroyed_events[0] = 0; } else if (which_l == 1) { watcher_id = PyCode_AddWatcher(second_code_object_callback); + code_watcher_ids[1] = watcher_id; num_code_object_created_events[1] = 0; num_code_object_destroyed_events[1] = 0; } @@ -375,9 +378,14 @@ clear_code_watcher(PyObject *self, PyObject *watcher_id) return NULL; } // reset static events counters - if (watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS) { - num_code_object_created_events[watcher_id_l] = 0; - num_code_object_destroyed_events[watcher_id_l] = 0; + if (watcher_id_l >= 0) { + for (int i = 0; i < NUM_CODE_WATCHERS; i++) { + if (watcher_id_l == code_watcher_ids[i]) { + code_watcher_ids[i] = -1; + num_code_object_created_events[i] = 0; + num_code_object_destroyed_events[i] = 0; + } + } } Py_RETURN_NONE; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 099f20b5a1b433..66815c72ffbc63 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -429,6 +429,7 @@ Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - code_watcher_ids - Modules/_testcapi/watchers.c - num_code_object_created_events - Modules/_testcapi/watchers.c - num_code_object_destroyed_events - Modules/_testcapi/watchers.c - pyfunc_watchers - From f50c17243a87b02086000185f6ed1cad4b8c2376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mina=20Gali=C4=87?= Date: Thu, 10 Aug 2023 01:47:46 +0100 Subject: [PATCH 094/598] GH-107812: extend `socket`'s netlink support to FreeBSD (gh-107813) --- ...2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst | 1 + Modules/socketmodule.h | 2 ++ configure | 15 +++++++++++++++ configure.ac | 3 ++- pyconfig.h.in | 3 +++ 5 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst diff --git a/Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst b/Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst new file mode 100644 index 00000000000000..0aac44fb418836 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst @@ -0,0 +1 @@ +Extend socket's netlink support to the FreeBSD platform. diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 663ae3d6e0dd6c..47146a28e02c8f 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -100,6 +100,8 @@ typedef int socklen_t; # include # endif # include +#elif defined(HAVE_NETLINK_NETLINK_H) +# include #else # undef AF_NETLINK #endif diff --git a/configure b/configure index 80b4a001c6d6d7..aaacf8d2669c16 100755 --- a/configure +++ b/configure @@ -11153,6 +11153,7 @@ fi # On Linux, netlink.h requires asm/types.h +# On FreeBSD, netlink.h is located in netlink/netlink.h ac_fn_c_check_header_compile "$LINENO" "linux/netlink.h" "ac_cv_header_linux_netlink_h" " #ifdef HAVE_ASM_TYPES_H #include @@ -11167,6 +11168,20 @@ then : printf "%s\n" "#define HAVE_LINUX_NETLINK_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "netlink/netlink.h" "ac_cv_header_netlink_netlink_h" " +#ifdef HAVE_ASM_TYPES_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif + +" +if test "x$ac_cv_header_netlink_netlink_h" = xyes +then : + printf "%s\n" "#define HAVE_NETLINK_NETLINK_H 1" >>confdefs.h + +fi # On Linux, qrtr.h requires asm/types.h diff --git a/configure.ac b/configure.ac index 8a84eaf6b370a1..ddf6da0b9da123 100644 --- a/configure.ac +++ b/configure.ac @@ -2880,7 +2880,8 @@ AC_CHECK_HEADERS([net/if.h], [], [], ]) # On Linux, netlink.h requires asm/types.h -AC_CHECK_HEADERS([linux/netlink.h], [], [], [ +# On FreeBSD, netlink.h is located in netlink/netlink.h +AC_CHECK_HEADERS([linux/netlink.h netlink/netlink.h], [], [], [ #ifdef HAVE_ASM_TYPES_H #include #endif diff --git a/pyconfig.h.in b/pyconfig.h.in index dab8ebf64371f3..181dc3d7d11370 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -841,6 +841,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_NETINET_IN_H +/* Define to 1 if you have the header file. */ +#undef HAVE_NETLINK_NETLINK_H + /* Define to 1 if you have the header file. */ #undef HAVE_NETPACKET_PACKET_H From 4890bfe1f906202ef521ffd327cae36e1afa0873 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 9 Aug 2023 18:05:51 -0700 Subject: [PATCH 095/598] Update README for the cases generator (#107826) --- Tools/cases_generator/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index fc9331656fe787..ed802e44f31ad5 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -7,10 +7,14 @@ What's currently here: - `lexer.py`: lexer for C, originally written by Mark Shannon - `plexer.py`: OO interface on top of lexer.py; main class: `PLexer` -- `parser.py`: Parser for instruction definition DSL; main class `Parser` +- `parsing.py`: Parser for instruction definition DSL; main class `Parser` - `generate_cases.py`: driver script to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `test_generator.py`: tests, require manual running using `pytest` +- `analysis.py`: `Analyzer` class used to read the input files +- `flags.py`: abstractions related to metadata flags for instructions +- `formatting.py`: `Formatter` class used to write the output files +- `instructions.py`: classes to analyze and write instructions +- `stacking.py`: code to handle generalized stack effects Note that there is some dummy C code at the top and bottom of `Python/bytecodes.c` From 0f2fb6efb4d5d8ca43a0e779b89a8e805880e09a Mon Sep 17 00:00:00 2001 From: Kostya Farber <73378227+kostyafarber@users.noreply.github.com> Date: Thu, 10 Aug 2023 07:39:14 +0100 Subject: [PATCH 096/598] gh-107689: Add docstring to `ctypes.Array` (#107697) --- Modules/_ctypes/_ctypes.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 9aee37a9d954ef..dc80291d3b810b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4793,6 +4793,16 @@ static PyMappingMethods Array_as_mapping = { Array_ass_subscript, }; +PyDoc_STRVAR(array_doc, +"Abstract base class for arrays.\n" +"\n" +"The recommended way to create concrete array types is by multiplying any\n" +"ctypes data type with a non-negative integer. Alternatively, you can subclass\n" +"this type and define _length_ and _type_ class variables. Array elements can\n" +"be read and written using standard subscript and slice accesses for slice\n" +"reads, the resulting object is not itself an Array." +); + PyTypeObject PyCArray_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_ctypes.Array", @@ -4813,8 +4823,8 @@ PyTypeObject PyCArray_Type = { 0, /* tp_getattro */ 0, /* tp_setattro */ &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + array_doc, /* tp_doc */ (traverseproc)PyCData_traverse, /* tp_traverse */ (inquiry)PyCData_clear, /* tp_clear */ 0, /* tp_richcompare */ From 4845b9712f2c187743344eca43fa1fb896bddfd6 Mon Sep 17 00:00:00 2001 From: denballakh <47365157+denballakh@users.noreply.github.com> Date: Thu, 10 Aug 2023 11:55:49 +0500 Subject: [PATCH 097/598] gh-107409: set `__wrapped__` attribute in `reprlib.recursive_repr` (#107410) Co-authored-by: Kumar Aditya --- Lib/reprlib.py | 1 + Lib/test/test_reprlib.py | 9 +++++++++ .../2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst | 1 + 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst diff --git a/Lib/reprlib.py b/Lib/reprlib.py index a92b3e3dbb613a..840dd0e20132b1 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -29,6 +29,7 @@ def wrapper(self): wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__wrapped__ = user_function return wrapper return decorating_function diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index e7216d427200c1..502287b620d066 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -765,5 +765,14 @@ def test_assigned_attributes(self): for name in assigned: self.assertIs(getattr(wrapper, name), getattr(wrapped, name)) + def test__wrapped__(self): + class X: + def __repr__(self): + return 'X()' + f = __repr__ # save reference to check it later + __repr__ = recursive_repr()(__repr__) + + self.assertIs(X.f, X.__repr__.__wrapped__) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst b/Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst new file mode 100644 index 00000000000000..1ecc7207605c70 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst @@ -0,0 +1 @@ +Set :attr:`!__wrapped__` attribute in :func:`reprlib.recursive_repr`. From 39ef93edb9802dccdb6555d4209ac2e60875a011 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 10 Aug 2023 09:19:05 +0200 Subject: [PATCH 098/598] gh-95065: Argument Clinic: Add functional tests of deprecated positionals (#107768) Move the "deprecated positinal" tests from clinic.test.c to _testclinic.c. Mock PY_VERSION_HEX in order to prevent generated compiler warnings/errors to trigger. Put clinic code for deprecated positionals in Modules/clinic/_testclinic_depr_star.c.h for easy inspection of the generated code. Co-authored-by: Alex Waygood Co-authored-by: Serhiy Storchaka --- Lib/test/clinic.test.c | 1161 ------------------- Lib/test/test_clinic.py | 103 ++ Modules/_testclinic.c | 260 ++++- Modules/clinic/_testclinic_depr_star.c.h | 974 ++++++++++++++++ Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 + 5 files changed, 1338 insertions(+), 1162 deletions(-) create mode 100644 Modules/clinic/_testclinic_depr_star.c.h diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index c7063e1139b0a9..019dc10073e986 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5467,1164 +5467,3 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) /*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ - - -/*[clinic input] -test_deprecate_positional_pos1_len1_optional - a: object - * [from 3.14] - b: object = None -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos1_len1_optional__doc__, -"test_deprecate_positional_pos1_len1_optional($module, /, a, b=None)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to\n" -"test_deprecate_positional_pos1_len1_optional() is deprecated.\n" -"Parameter \'b\' will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_OPTIONAL_METHODDEF \ - {"test_deprecate_positional_pos1_len1_optional", _PyCFunction_CAST(test_deprecate_positional_pos1_len1_optional), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1_optional__doc__}, - -static PyObject * -test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, - PyObject *a, PyObject *b); - -static PyObject * -test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos1_len1_optional", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *a; - PyObject *b = Py_None; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1_optional' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1_optional' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1_optional' to be " \ - "keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to " - "test_deprecate_positional_pos1_len1_optional() is deprecated. " - "Parameter 'b' will become a keyword-only parameter in Python " - "3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - if (!noptargs) { - goto skip_optional_pos; - } - b = args[1]; -skip_optional_pos: - return_value = test_deprecate_positional_pos1_len1_optional_impl(module, a, b); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, - PyObject *a, PyObject *b) -/*[clinic end generated code: output=144cbf1adc574dd9 input=89099f3dacd757da]*/ - - -/*[clinic input] -test_deprecate_positional_pos1_len1 - a: object - * [from 3.14] - b: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos1_len1__doc__, -"test_deprecate_positional_pos1_len1($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to\n" -"test_deprecate_positional_pos1_len1() is deprecated. Parameter \'b\'\n" -"will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_METHODDEF \ - {"test_deprecate_positional_pos1_len1", _PyCFunction_CAST(test_deprecate_positional_pos1_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1__doc__}, - -static PyObject * -test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, - PyObject *b); - -static PyObject * -test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos1_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1' to be keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to " - "test_deprecate_positional_pos1_len1() is deprecated. Parameter " - "'b' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = test_deprecate_positional_pos1_len1_impl(module, a, b); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, - PyObject *b) -/*[clinic end generated code: output=994bd57c1c634709 input=1702bbab1e9b3b99]*/ - - -/*[clinic input] -test_deprecate_positional_pos1_len2_with_kwd - a: object - * [from 3.14] - b: object - c: object - * - d: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos1_len2_with_kwd__doc__, -"test_deprecate_positional_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" -"--\n" -"\n" -"Note: Passing more than 1 positional argument to\n" -"test_deprecate_positional_pos1_len2_with_kwd() is deprecated.\n" -"Parameters \'b\' and \'c\' will become keyword-only parameters in Python\n" -"3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS1_LEN2_WITH_KWD_METHODDEF \ - {"test_deprecate_positional_pos1_len2_with_kwd", _PyCFunction_CAST(test_deprecate_positional_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len2_with_kwd__doc__}, - -static PyObject * -test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, - PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos1_len2_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 1 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to " - "test_deprecate_positional_pos1_len2_with_kwd() is deprecated. " - "Parameters 'b' and 'c' will become keyword-only parameters in " - "Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = test_deprecate_positional_pos1_len2_with_kwd_impl(module, a, b, c, d); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, - PyObject *a, PyObject *b, - PyObject *c, PyObject *d) -/*[clinic end generated code: output=146c60ecbcdbf4b8 input=28cdb885f6c34eab]*/ - - -/*[clinic input] -test_deprecate_positional_pos0_len1 - * [from 3.14] - a: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos0_len1__doc__, -"test_deprecate_positional_pos0_len1($module, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"test_deprecate_positional_pos0_len1() is deprecated. Parameter \'a\'\n" -"will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS0_LEN1_METHODDEF \ - {"test_deprecate_positional_pos0_len1", _PyCFunction_CAST(test_deprecate_positional_pos0_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len1__doc__}, - -static PyObject * -test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a); - -static PyObject * -test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos0_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'test_deprecate_positional_pos0_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'test_deprecate_positional_pos0_len1' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'test_deprecate_positional_pos0_len1' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "test_deprecate_positional_pos0_len1() is deprecated. Parameter " - "'a' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = test_deprecate_positional_pos0_len1_impl(module, a); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=dce99971a2494f9f input=678206db25c0652c]*/ - - -/*[clinic input] -test_deprecate_positional_pos0_len2 - * [from 3.14] - a: object - b: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos0_len2__doc__, -"test_deprecate_positional_pos0_len2($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"test_deprecate_positional_pos0_len2() is deprecated. Parameters \'a\'\n" -"and \'b\' will become keyword-only parameters in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS0_LEN2_METHODDEF \ - {"test_deprecate_positional_pos0_len2", _PyCFunction_CAST(test_deprecate_positional_pos0_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len2__doc__}, - -static PyObject * -test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, - PyObject *b); - -static PyObject * -test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos0_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'test_deprecate_positional_pos0_len2' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'test_deprecate_positional_pos0_len2' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'test_deprecate_positional_pos0_len2' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "test_deprecate_positional_pos0_len2() is deprecated. Parameters " - "'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = test_deprecate_positional_pos0_len2_impl(module, a, b); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, - PyObject *b) -/*[clinic end generated code: output=06999692e0c8dac4 input=fae0d0b1d480c939]*/ - - -/*[clinic input] -test_deprecate_positional_pos0_len3_with_kwdonly - * [from 3.14] - a: object - b: object - c: object - * - e: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos0_len3_with_kwdonly__doc__, -"test_deprecate_positional_pos0_len3_with_kwdonly($module, /, a, b, c,\n" -" *, e)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated.\n" -"Parameters \'a\', \'b\' and \'c\' will become keyword-only parameters in\n" -"Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS0_LEN3_WITH_KWDONLY_METHODDEF \ - {"test_deprecate_positional_pos0_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos0_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len3_with_kwdonly__doc__}, - -static PyObject * -test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *e); - -static PyObject * -test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(e), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "e", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos0_len3_with_kwdonly", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *e; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of " \ - "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of " \ - "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of " \ - "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "test_deprecate_positional_pos0_len3_with_kwdonly() is " - "deprecated. Parameters 'a', 'b' and 'c' will become keyword-only" - " parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - e = args[3]; - return_value = test_deprecate_positional_pos0_len3_with_kwdonly_impl(module, a, b, c, e); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *e) -/*[clinic end generated code: output=a553e33101dc42b2 input=1b0121770c0c52e0]*/ - - -/*[clinic input] -test_deprecate_positional_pos2_len1 - a: object - b: object - * [from 3.14] - c: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos2_len1__doc__, -"test_deprecate_positional_pos2_len1($module, /, a, b, c)\n" -"--\n" -"\n" -"Note: Passing 3 positional arguments to\n" -"test_deprecate_positional_pos2_len1() is deprecated. Parameter \'c\'\n" -"will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS2_LEN1_METHODDEF \ - {"test_deprecate_positional_pos2_len1", _PyCFunction_CAST(test_deprecate_positional_pos2_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len1__doc__}, - -static PyObject * -test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c); - -static PyObject * -test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos2_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - PyObject *a; - PyObject *b; - PyObject *c; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ - " 'test_deprecate_positional_pos2_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ - " 'test_deprecate_positional_pos2_len1' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ - " 'test_deprecate_positional_pos2_len1' to be keyword-only." - # endif - #endif - if (nargs == 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 3 positional arguments to " - "test_deprecate_positional_pos2_len1() is deprecated. Parameter " - "'c' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - return_value = test_deprecate_positional_pos2_len1_impl(module, a, b, c); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c) -/*[clinic end generated code: output=f96454a4970b443c input=e1d129689e69ec7c]*/ - - -/*[clinic input] -test_deprecate_positional_pos2_len2 - a: object - b: object - * [from 3.14] - c: object - d: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos2_len2__doc__, -"test_deprecate_positional_pos2_len2($module, /, a, b, c, d)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"test_deprecate_positional_pos2_len2() is deprecated. Parameters \'c\'\n" -"and \'d\' will become keyword-only parameters in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS2_LEN2_METHODDEF \ - {"test_deprecate_positional_pos2_len2", _PyCFunction_CAST(test_deprecate_positional_pos2_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len2__doc__}, - -static PyObject * -test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c, - PyObject *d); - -static PyObject * -test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos2_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len2' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len2' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len2' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "test_deprecate_positional_pos2_len2() is deprecated. Parameters " - "'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = test_deprecate_positional_pos2_len2_impl(module, a, b, c, d); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c, - PyObject *d) -/*[clinic end generated code: output=5e648e887da0a804 input=0d53533463a12792]*/ - - -/*[clinic input] -test_deprecate_positional_pos2_len3_with_kwdonly - a: object - b: object - * [from 3.14] - c: object - d: object - * - e: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos2_len3_with_kwdonly__doc__, -"test_deprecate_positional_pos2_len3_with_kwdonly($module, /, a, b, c,\n" -" d, *, e)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated.\n" -"Parameters \'c\' and \'d\' will become keyword-only parameters in Python\n" -"3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS2_LEN3_WITH_KWDONLY_METHODDEF \ - {"test_deprecate_positional_pos2_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos2_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len3_with_kwdonly__doc__}, - -static PyObject * -test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *d, - PyObject *e); - -static PyObject * -test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 5 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos2_len3_with_kwdonly", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[5]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - PyObject *e; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ - "be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ - "be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ - "be keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "test_deprecate_positional_pos2_len3_with_kwdonly() is " - "deprecated. Parameters 'c' and 'd' will become keyword-only " - "parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - e = args[4]; - return_value = test_deprecate_positional_pos2_len3_with_kwdonly_impl(module, a, b, c, d, e); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *d, - PyObject *e) -/*[clinic end generated code: output=383d56b03f7c2dcb input=154fd450448d8935]*/ - - -/*[clinic input] -@classmethod -Test.__new__ - * [from 3.14] - a: object -The deprecation message should use the class name instead of __new__. -[clinic start generated code]*/ - -PyDoc_STRVAR(Test__doc__, -"Test(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __new__.\n" -"\n" -"Note: Passing positional arguments to Test() is deprecated. Parameter\n" -"\'a\' will become a keyword-only parameter in Python 3.14.\n" -""); - -static PyObject * -Test_impl(PyTypeObject *type, PyObject *a); - -static PyObject * -Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "Test", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'Test.__new__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'Test.__new__' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'Test.__new__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to Test() is deprecated. Parameter " - "'a' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = Test_impl(type, a); - -exit: - return return_value; -} - -static PyObject * -Test_impl(PyTypeObject *type, PyObject *a) -/*[clinic end generated code: output=d15a69ea37ec6502 input=f133dc077aef49ec]*/ - - -/*[clinic input] -m.T.__init__ - * [from 3.14] - a: object -The deprecation message should use the class name instead of __init__. -[clinic start generated code]*/ - -PyDoc_STRVAR(m_T___init____doc__, -"T(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __init__.\n" -"\n" -"Note: Passing positional arguments to m.T() is deprecated. Parameter\n" -"\'a\' will become a keyword-only parameter in Python 3.14.\n" -""); - -static int -m_T___init___impl(TestObj *self, PyObject *a); - -static int -m_T___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "T", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'm.T.__init__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'm.T.__init__' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'm.T.__init__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to m.T() is deprecated. Parameter " - "'a' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = m_T___init___impl((TestObj *)self, a); - -exit: - return return_value; -} - -static int -m_T___init___impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=ef43c425816a549f input=f71b51dbe19fa657]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3921523af067f8..a649b5fe2201c8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2,6 +2,7 @@ # Copyright 2012-2013 by Larry Hastings. # Licensed to the PSF under a contributor agreement. +from functools import partial from test import support, test_tools from test.support import os_helper from test.support.os_helper import TESTFN, unlink @@ -2431,6 +2432,19 @@ class ClinicFunctionalTest(unittest.TestCase): locals().update((name, getattr(ac_tester, name)) for name in dir(ac_tester) if name.startswith('test_')) + def check_depr_star(self, pnames, fn, *args, **kwds): + regex = ( + fr"Passing( more than)?( [0-9]+)? positional argument(s)? to " + fr"{fn.__name__}\(\) is deprecated. Parameter(s)? {pnames} will " + fr"become( a)? keyword-only parameter(s)? in Python 3\.14" + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + def test_objects_converter(self): with self.assertRaises(TypeError): ac_tester.objects_converter() @@ -2895,6 +2909,95 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) + def test_depr_star_new(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarNew() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + ac_tester.DeprStarNew(None) + self.assertEqual(cm.filename, __file__) + + def test_depr_star_init(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarInit() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + ac_tester.DeprStarInit(None) + self.assertEqual(cm.filename, __file__) + + def test_depr_star_pos0_len1(self): + fn = ac_tester.depr_star_pos0_len1 + fn(a=None) + self.check_depr_star("'a'", fn, "a") + + def test_depr_star_pos0_len2(self): + fn = ac_tester.depr_star_pos0_len2 + fn(a=0, b=0) + check = partial(self.check_depr_star, "'a' and 'b'", fn) + check("a", b=0) + check("a", "b") + + def test_depr_star_pos0_len3_with_kwd(self): + fn = ac_tester.depr_star_pos0_len3_with_kwd + fn(a=0, b=0, c=0, d=0) + check = partial(self.check_depr_star, "'a', 'b' and 'c'", fn) + check("a", b=0, c=0, d=0) + check("a", "b", c=0, d=0) + check("a", "b", "c", d=0) + + def test_depr_star_pos1_len1_opt(self): + fn = ac_tester.depr_star_pos1_len1_opt + fn(a=0, b=0) + fn("a", b=0) + fn(a=0) # b is optional + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len1(self): + fn = ac_tester.depr_star_pos1_len1 + fn(a=0, b=0) + fn("a", b=0) + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len2_with_kwd(self): + fn = ac_tester.depr_star_pos1_len2_with_kwd + fn(a=0, b=0, c=0, d=0), + fn("a", b=0, c=0, d=0), + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b", c=0, d=0), + check("a", "b", "c", d=0), + + def test_depr_star_pos2_len1(self): + fn = ac_tester.depr_star_pos2_len1 + fn(a=0, b=0, c=0) + fn("a", b=0, c=0) + fn("a", "b", c=0) + check = partial(self.check_depr_star, "'c'", fn) + check("a", "b", "c") + + def test_depr_star_pos2_len2(self): + fn = ac_tester.depr_star_pos2_len2 + fn(a=0, b=0, c=0, d=0) + fn("a", b=0, c=0, d=0) + fn("a", "b", c=0, d=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0) + check("a", "b", "c", "d") + + def test_depr_star_pos2_len2_with_kwd(self): + fn = ac_tester.depr_star_pos2_len2_with_kwd + fn(a=0, b=0, c=0, d=0, e=0) + fn("a", b=0, c=0, d=0, e=0) + fn("a", "b", c=0, d=0, e=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0, e=0) + check("a", "b", "c", "d", e=0) + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 8ba8f8ecadec7a..8fa3cc83d871b1 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1193,6 +1193,240 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) } +/*[clinic input] +output push +destination deprstar new file '{dirname}/clinic/_testclinic_depr_star.c.h' +output everything deprstar +#output methoddef_ifndef buffer 1 +output docstring_prototype suppress +output parser_prototype suppress +output impl_definition block +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f88f37038e00fb0a]*/ + + +// Mock Python version 3.8 +#define _SAVED_PY_VERSION PY_VERSION_HEX +#undef PY_VERSION_HEX +#define PY_VERSION_HEX 0x03080000 + + +#include "clinic/_testclinic_depr_star.c.h" + + +/*[clinic input] +class _testclinic.DeprStarNew "PyObject *" "PyObject" +@classmethod +_testclinic.DeprStarNew.__new__ as depr_star_new + * [from 3.14] + a: object +The deprecation message should use the class name instead of __new__. +[clinic start generated code]*/ + +static PyObject * +depr_star_new_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=bdbb36244f90cf46 input=f4ae7dafbc23c378]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject DeprStarNew = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprStarNew", + .tp_basicsize = sizeof(PyObject), + .tp_new = depr_star_new, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprStarInit "PyObject *" "PyObject" +_testclinic.DeprStarInit.__init__ as depr_star_init + * [from 3.14] + a: object +The deprecation message should use the class name instead of __init__. +[clinic start generated code]*/ + +static int +depr_star_init_impl(PyObject *self, PyObject *a) +/*[clinic end generated code: output=8d27b43c286d3ecc input=659ebc748d87fa86]*/ +{ + return 0; +} + +static PyTypeObject DeprStarInit = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprStarInit", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_star_init, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +depr_star_pos0_len1 + * [from 3.14] + a: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos0_len1_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=e1c6c2b423129499 input=089b9aee25381b69]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos0_len2 + * [from 3.14] + a: object + b: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=96df9be39859c7e4 input=65c83a32e01495c6]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos0_len3_with_kwd + * [from 3.14] + a: object + b: object + c: object + * + d: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=7f2531eda837052f input=b33f620f57d9270f]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos1_len1_opt + a: object + * [from 3.14] + b: object = None +[clinic start generated code]*/ + +static PyObject * +depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=b5b4e326ee3b216f input=4a4b8ff72eae9ff7]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos1_len1 + a: object + * [from 3.14] + b: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=eab92e37d5b0a480 input=1e7787a9fe5f62a0]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos1_len2_with_kwd + a: object + * [from 3.14] + b: object + c: object + * + d: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=3bccab672b7cfbb8 input=6bc7bd742fa8be15]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos2_len1 + a: object + b: object + * [from 3.14] + c: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=20f5b230e9beeb70 input=5fc3e1790dec00d5]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos2_len2 + a: object + b: object + * [from 3.14] + c: object + d: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=9f90ed8fbce27d7a input=9cc8003b89d38779]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos2_len2_with_kwd + a: object + b: object + * [from 3.14] + c: object + d: object + * + e: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=05432c4f20527215 input=831832d90534da91]*/ +{ + Py_RETURN_NONE; +} + + +// Reset PY_VERSION_HEX +#undef PY_VERSION_HEX +#define PY_VERSION_HEX _SAVED_PY_VERSION +#undef _SAVED_PY_VERSION + + +/*[clinic input] +output pop +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e7c7c42daced52b0]*/ + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -1248,6 +1482,16 @@ static PyMethodDef tester_methods[] = { CLONE_F2_METHODDEF CLONE_WITH_CONV_F1_METHODDEF CLONE_WITH_CONV_F2_METHODDEF + + DEPR_STAR_POS0_LEN1_METHODDEF + DEPR_STAR_POS0_LEN2_METHODDEF + DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF + DEPR_STAR_POS1_LEN1_OPT_METHODDEF + DEPR_STAR_POS1_LEN1_METHODDEF + DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF + DEPR_STAR_POS2_LEN1_METHODDEF + DEPR_STAR_POS2_LEN2_METHODDEF + DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF {NULL, NULL} }; @@ -1261,7 +1505,21 @@ static struct PyModuleDef _testclinic_module = { PyMODINIT_FUNC PyInit__testclinic(void) { - return PyModule_Create(&_testclinic_module); + PyObject *m = PyModule_Create(&_testclinic_module); + if (m == NULL) { + return NULL; + } + if (PyModule_AddType(m, &DeprStarNew) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprStarInit) < 0) { + goto error; + } + return m; + +error: + Py_DECREF(m); + return NULL; } #undef RETURN_PACKED_ARGS diff --git a/Modules/clinic/_testclinic_depr_star.c.h b/Modules/clinic/_testclinic_depr_star.c.h new file mode 100644 index 00000000000000..0c2fa088268c6d --- /dev/null +++ b/Modules/clinic/_testclinic_depr_star.c.h @@ -0,0 +1,974 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(depr_star_new__doc__, +"DeprStarNew(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static PyObject * +depr_star_new_impl(PyTypeObject *type, PyObject *a); + +static PyObject * +depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.__new__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.__new__' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.__new__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = depr_star_new_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_init__doc__, +"DeprStarInit(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static int +depr_star_init_impl(PyObject *self, PyObject *a); + +static int +depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.__init__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.__init__' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.__init__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarInit() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = depr_star_init_impl(self, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len1__doc__, +"depr_star_pos0_len1($module, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len1() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN1_METHODDEF \ + {"depr_star_pos0_len1", _PyCFunction_CAST(depr_star_pos0_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len1__doc__}, + +static PyObject * +depr_star_pos0_len1_impl(PyObject *module, PyObject *a); + +static PyObject * +depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " 'depr_star_pos0_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " 'depr_star_pos0_len1' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " 'depr_star_pos0_len1' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len1() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_pos0_len1_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len2__doc__, +"depr_star_pos0_len2($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len2() is\n" +"deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" +"in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN2_METHODDEF \ + {"depr_star_pos0_len2", _PyCFunction_CAST(depr_star_pos0_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len2__doc__}, + +static PyObject * +depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'depr_star_pos0_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'depr_star_pos0_len2' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'depr_star_pos0_len2' to be keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len2() is " + "deprecated. Parameters 'a' and 'b' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos0_len2_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, +"depr_star_pos0_len3_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" +"is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" +"parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ + {"depr_star_pos0_len3_with_kwd", _PyCFunction_CAST(depr_star_pos0_len3_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len3_with_kwd__doc__}, + +static PyObject * +depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len3_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ + "keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ + "keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ + "keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len3_with_kwd() " + "is deprecated. Parameters 'a', 'b' and 'c' will become " + "keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos0_len3_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, +"depr_star_pos1_len1_opt($module, /, a, b=None)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ + {"depr_star_pos1_len1_opt", _PyCFunction_CAST(depr_star_pos1_len1_opt), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1_opt__doc__}, + +static PyObject * +depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1_opt' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1_opt' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1_opt' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = depr_star_pos1_len1_opt_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1__doc__, +"depr_star_pos1_len1($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_METHODDEF \ + {"depr_star_pos1_len1", _PyCFunction_CAST(depr_star_pos1_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1__doc__}, + +static PyObject * +depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos1_len1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, +"depr_star_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to\n" +"depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos1_len2_with_kwd", _PyCFunction_CAST(depr_star_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." + # endif + #endif + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" + " 'c' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos1_len2_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len1__doc__, +"depr_star_pos2_len1($module, /, a, b, c)\n" +"--\n" +"\n" +"Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" +"deprecated. Parameter \'c\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN1_METHODDEF \ + {"depr_star_pos2_len1", _PyCFunction_CAST(depr_star_pos2_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len1__doc__}, + +static PyObject * +depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ + " 'depr_star_pos2_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ + " 'depr_star_pos2_len1' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ + " 'depr_star_pos2_len1' to be keyword-only." + # endif + #endif + if (nargs == 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 3 positional arguments to depr_star_pos2_len1() is " + "deprecated. Parameter 'c' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = depr_star_pos2_len1_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2__doc__, +"depr_star_pos2_len2($module, /, a, b, c, d)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" +"become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_METHODDEF \ + {"depr_star_pos2_len2", _PyCFunction_CAST(depr_star_pos2_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2__doc__}, + +static PyObject * +depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" + " become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos2_len2_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, +"depr_star_pos2_len2_with_kwd($module, /, a, b, c, d, *, e)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos2_len2_with_kwd", _PyCFunction_CAST(depr_star_pos2_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" + " 'd' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + return_value = depr_star_pos2_len2_with_kwd_impl(module, a, b, c, d, e); + +exit: + return return_value; +} +/*[clinic end generated code: output=18ab056f6cc06d7e input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index dedcb3141457c0..28bd2a4430d14e 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -322,6 +322,8 @@ Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - Modules/_testcapi/vectorcall.c - MethodDescriptor2_Type - +Modules/_testclinic.c - DeprStarInit - +Modules/_testclinic.c - DeprStarNew - ################################## From 494e3d4436774a5ac1a569a635b8c5c881ef1c0c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Aug 2023 12:29:06 +0100 Subject: [PATCH 099/598] GH-107774: Add missing audit event for PEP 669 (GH-107775) --- Lib/test/audit-tests.py | 11 +++++++++++ Lib/test/test_audit.py | 13 +++++++++++++ .../2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst | 3 +++ Python/instrumentation.c | 3 +++ 4 files changed, 30 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 0edc9d9c472766..9504829e96f00e 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -514,6 +514,17 @@ def test_not_in_gc(): assert hook not in o +def test_sys_monitoring_register_callback(): + import sys + + def hook(event, args): + if event.startswith("sys.monitoring"): + print(event, args) + + sys.addaudithook(hook) + sys.monitoring.register_callback(1, 1, None) + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 0b69864751d83d..b12ffa5d872e83 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -257,5 +257,18 @@ def test_not_in_gc(self): self.fail(stderr) + def test_sys_monitoring_register_callback(self): + returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys.monitoring.register_callback", "(None,)")] + + self.assertEqual(actual, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst b/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst new file mode 100644 index 00000000000000..b89b50c79f7e2a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst @@ -0,0 +1,3 @@ +PEP 669 specifies that ``sys.monitoring.register_callback`` will generate an +audit event. Pre-releases of Python 3.12 did not generate the audit event. +This is now fixed. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 65ea7902a80a60..64684ad522f687 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1851,6 +1851,9 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, PyErr_Format(PyExc_ValueError, "invalid event %d", event); return NULL; } + if (PySys_Audit("sys.monitoring.register_callback", "O", func) < 0) { + return NULL; + } if (func == Py_None) { func = NULL; } From bafedfbebd0f21291a7824d54e39ec79f142428b Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 10 Aug 2023 13:03:47 +0100 Subject: [PATCH 100/598] gh-106149: move CFG and basicblock definitions into flowgraph.c, use them as opaque types in compile.c (#107639) --- Include/internal/pycore_compile.h | 5 + Include/internal/pycore_flowgraph.h | 86 +----- Python/compile.c | 432 +++++---------------------- Python/flowgraph.c | 437 +++++++++++++++++++++++++--- 4 files changed, 482 insertions(+), 478 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index fa2f64021e73c9..ad657c0f0fcedc 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -54,6 +54,11 @@ typedef struct { int s_next_free_label; /* next free label id */ } _PyCompile_InstructionSequence; +int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); +int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, + int opcode, int oparg, + _PyCompilerSrcLocation loc); + typedef struct { PyObject *u_name; PyObject *u_qualname; /* dot-separated qualified name (lazy) */ diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index cb55f8712e7ad6..58fed46886ea45 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -11,88 +11,26 @@ extern "C" { #include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int i_opcode; - int i_oparg; - _PyCompilerSrcLocation i_loc; - struct _PyCfgBasicblock_ *i_target; /* target block (if jump instruction) */ - struct _PyCfgBasicblock_ *i_except; /* target block when exception is raised */ -} _PyCfgInstruction; - typedef struct { int id; } _PyCfgJumpTargetLabel; +struct _PyCfgBuilder; -typedef struct { - struct _PyCfgBasicblock_ *handlers[CO_MAXBLOCKS+1]; - int depth; -} _PyCfgExceptStack; - -typedef struct _PyCfgBasicblock_ { - /* Each basicblock in a compilation unit is linked via b_list in the - reverse order that the block are allocated. b_list points to the next - block in this list, not to be confused with b_next, which is next by - control flow. */ - struct _PyCfgBasicblock_ *b_list; - /* The label of this block if it is a jump target, -1 otherwise */ - _PyCfgJumpTargetLabel b_label; - /* Exception stack at start of block, used by assembler to create the exception handling table */ - _PyCfgExceptStack *b_exceptstack; - /* pointer to an array of instructions, initially NULL */ - _PyCfgInstruction *b_instr; - /* If b_next is non-NULL, it is a pointer to the next - block reached by normal control flow. */ - struct _PyCfgBasicblock_ *b_next; - /* number of instructions used */ - int b_iused; - /* length of instruction array (b_instr) */ - int b_ialloc; - /* Used by add_checks_for_loads_of_unknown_variables */ - uint64_t b_unsafe_locals_mask; - /* Number of predecessors that a block has. */ - int b_predecessors; - /* depth of stack upon entry of block, computed by stackdepth() */ - int b_startdepth; - /* Basic block is an exception handler that preserves lasti */ - unsigned b_preserve_lasti : 1; - /* Used by compiler passes to mark whether they have visited a basic block. */ - unsigned b_visited : 1; - /* b_except_handler is used by the cold-detection algorithm to mark exception targets */ - unsigned b_except_handler : 1; - /* b_cold is true if this block is not perf critical (like an exception handler) */ - unsigned b_cold : 1; - /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */ - unsigned b_warm : 1; -} _PyCfgBasicblock; +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); -int _PyBasicblock_InsertInstruction(_PyCfgBasicblock *block, int pos, _PyCfgInstruction *instr); +struct _PyCfgBuilder* _PyCfgBuilder_New(void); +void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); +int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); -typedef struct cfg_builder_ { - /* The entryblock, at which control flow begins. All blocks of the - CFG are reachable through the b_next links */ - _PyCfgBasicblock *g_entryblock; - /* Pointer to the most recently allocated block. By following - b_list links, you can reach all allocated blocks. */ - _PyCfgBasicblock *g_block_list; - /* pointer to the block currently being constructed */ - _PyCfgBasicblock *g_curblock; - /* label for the next instruction to be placed */ - _PyCfgJumpTargetLabel g_current_label; -} _PyCfgBuilder; - -int _PyCfgBuilder_UseLabel(_PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(_PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); - -int _PyCfgBuilder_Init(_PyCfgBuilder *g); -void _PyCfgBuilder_Fini(_PyCfgBuilder *g); - -int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, +int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); -int _PyCfg_Stackdepth(_PyCfgBuilder *g); -void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock); -int _PyCfg_ResolveJumps(_PyCfgBuilder *g); + +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, + int code_flags, int *stackdepth, int *nlocalsplus, + _PyCompile_InstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, diff --git a/Python/compile.c b/Python/compile.c index 68af5d61ac8de0..83cf45550e2588 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -70,9 +70,7 @@ && ((C)->u->u_ste->ste_type == ModuleBlock)) typedef _PyCompilerSrcLocation location; -typedef _PyCfgInstruction cfg_instr; -typedef _PyCfgBasicblock basicblock; -typedef _PyCfgBuilder cfg_builder; +typedef struct _PyCfgBuilder cfg_builder; #define LOCATION(LNO, END_LNO, COL, END_COL) \ ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) @@ -101,7 +99,7 @@ static jump_target_label NO_LABEL = {-1}; } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(instr_sequence_use_label(INSTR_SEQUENCE(C), (LBL).id)) + RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) /* fblockinfo tracks the current frame block. @@ -218,8 +216,9 @@ instr_sequence_new_label(instr_sequence *seq) return lbl; } -static int -instr_sequence_use_label(instr_sequence *seq, int lbl) { +int +_PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl) +{ int old_size = seq->s_labelmap_size; RETURN_IF_ERROR( _PyCompile_EnsureArrayLargeEnough(lbl, @@ -238,8 +237,9 @@ instr_sequence_use_label(instr_sequence *seq, int lbl) { #define MAX_OPCODE 511 -static int -instr_sequence_addop(instr_sequence *seq, int opcode, int oparg, location loc) +int +_PyCompile_InstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, + location loc) { assert(0 <= opcode && opcode <= MAX_OPCODE); assert(IS_WITHIN_OPCODE_RANGE(opcode)); @@ -288,10 +288,12 @@ instr_sequence_fini(instr_sequence *seq) { seq->s_instrs = NULL; } -static int -instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { - memset(g, 0, sizeof(cfg_builder)); - RETURN_IF_ERROR(_PyCfgBuilder_Init(g)); +static cfg_builder* +instr_sequence_to_cfg(instr_sequence *seq) { + cfg_builder *g = _PyCfgBuilder_New(); + if (g == NULL) { + return NULL; + } /* There can be more than one label for the same offset. The * offset2lbl maping selects one of them which we use consistently. @@ -300,7 +302,7 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { int *offset2lbl = PyMem_Malloc(seq->s_used * sizeof(int)); if (offset2lbl == NULL) { PyErr_NoMemory(); - return ERROR; + goto error; } for (int i = 0; i < seq->s_used; i++) { offset2lbl[i] = -1; @@ -336,23 +338,17 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { goto error; } } - PyMem_Free(offset2lbl); - - int nblocks = 0; - for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { - nblocks++; - } - if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { - PyErr_NoMemory(); - return ERROR; + if (_PyCfgBuilder_CheckSize(g) < 0) { + goto error; } - return SUCCESS; + PyMem_Free(offset2lbl); + return g; error: + _PyCfgBuilder_Free(g); PyMem_Free(offset2lbl); - return ERROR; + return NULL; } - /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. */ @@ -920,7 +916,7 @@ codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!OPCODE_HAS_ARG(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return instr_sequence_addop(seq, opcode, 0, loc); + return _PyCompile_InstructionSequence_Addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -1153,7 +1149,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return instr_sequence_addop(seq, opcode, oparg_, loc); + return _PyCompile_InstructionSequence_Addop(seq, opcode, oparg_, loc); } static int @@ -1163,7 +1159,7 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return instr_sequence_addop(seq, opcode, target.id, loc); + return _PyCompile_InstructionSequence_Addop(seq, opcode, target.id, loc); } #define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ @@ -7492,201 +7488,6 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) return SUCCESS; } - -static int * -build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) -{ - int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); - - int noffsets = ncellvars + nfreevars; - int *fixed = PyMem_New(int, noffsets); - if (fixed == NULL) { - PyErr_NoMemory(); - return NULL; - } - for (int i = 0; i < noffsets; i++) { - fixed[i] = nlocals + i; - } - - PyObject *varname, *cellindex; - Py_ssize_t pos = 0; - while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); - if (varindex != NULL) { - assert(PyLong_AS_LONG(cellindex) < INT_MAX); - assert(PyLong_AS_LONG(varindex) < INT_MAX); - int oldindex = (int)PyLong_AS_LONG(cellindex); - int argoffset = (int)PyLong_AS_LONG(varindex); - fixed[oldindex] = argoffset; - } - } - - return fixed; -} - -#define IS_GENERATOR(CF) \ - ((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) - -static int -insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, - int *fixed, int nfreevars, int code_flags) -{ - assert(umd->u_firstlineno > 0); - - /* Add the generator prefix instructions. */ - if (IS_GENERATOR(code_flags)) { - /* Note that RETURN_GENERATOR + POP_TOP have a net stack effect - * of 0. This is because RETURN_GENERATOR pushes an element - * with _PyFrame_StackPush before switching stacks. - */ - cfg_instr make_gen = { - .i_opcode = RETURN_GENERATOR, - .i_oparg = 0, - .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), - .i_target = NULL, - }; - RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &make_gen)); - cfg_instr pop_top = { - .i_opcode = POP_TOP, - .i_oparg = 0, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; - RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 1, &pop_top)); - } - - /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - if (ncellvars) { - // umd->u_cellvars has the cells out of order so we sort them - // before adding the MAKE_CELL instructions. Note that we - // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); - int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); - if (sorted == NULL) { - PyErr_NoMemory(); - return ERROR; - } - for (int i = 0; i < ncellvars; i++) { - sorted[fixed[i]] = i + 1; - } - for (int i = 0, ncellsused = 0; ncellsused < ncellvars; i++) { - int oldindex = sorted[i] - 1; - if (oldindex == -1) { - continue; - } - cfg_instr make_cell = { - .i_opcode = MAKE_CELL, - // This will get fixed in offset_derefs(). - .i_oparg = oldindex, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; - if (_PyBasicblock_InsertInstruction(entryblock, ncellsused, &make_cell) < 0) { - PyMem_RawFree(sorted); - return ERROR; - } - ncellsused += 1; - } - PyMem_RawFree(sorted); - } - - if (nfreevars) { - cfg_instr copy_frees = { - .i_opcode = COPY_FREE_VARS, - .i_oparg = nfreevars, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; - RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, ©_frees)); - } - - return SUCCESS; -} - -static int -fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) -{ - int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); - int noffsets = ncellvars + nfreevars; - - // First deal with duplicates (arg cells). - int numdropped = 0; - for (int i = 0; i < noffsets ; i++) { - if (fixedmap[i] == i + nlocals) { - fixedmap[i] -= numdropped; - } - else { - // It was a duplicate (cell/arg). - numdropped += 1; - } - } - - // Then update offsets, either relative to locals or by cell2arg. - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *inst = &b->b_instr[i]; - // This is called before extended args are generated. - assert(inst->i_opcode != EXTENDED_ARG); - int oldoffset = inst->i_oparg; - switch(inst->i_opcode) { - case MAKE_CELL: - case LOAD_CLOSURE: - case LOAD_DEREF: - case STORE_DEREF: - case DELETE_DEREF: - case LOAD_FROM_DICT_OR_DEREF: - assert(oldoffset >= 0); - assert(oldoffset < noffsets); - assert(fixedmap[oldoffset] >= 0); - inst->i_oparg = fixedmap[oldoffset]; - } - } - } - - return numdropped; -} - - -static int -prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) -{ - assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); - assert(INT_MAX - nlocals - ncellvars > 0); - assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); - int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(umd); - if (cellfixedoffsets == NULL) { - return ERROR; - } - - - // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { - PyMem_Free(cellfixedoffsets); - return ERROR; - } - - int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); - PyMem_Free(cellfixedoffsets); // At this point we're done with it. - cellfixedoffsets = NULL; - if (numdropped < 0) { - return ERROR; - } - - nlocalsplus -= numdropped; - return nlocalsplus; -} - static int add_return_at_end(struct compiler *c, int addNone) { @@ -7700,12 +7501,11 @@ add_return_at_end(struct compiler *c, int addNone) return SUCCESS; } -static int cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq); - static PyCodeObject * optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, int code_flags, PyObject *filename) { + cfg_builder *g = NULL; instr_sequence optimized_instrs; memset(&optimized_instrs, 0, sizeof(instr_sequence)); @@ -7714,51 +7514,28 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - cfg_builder g; - if (instr_sequence_to_cfg(&u->u_instr_sequence, &g) < 0) { + g = instr_sequence_to_cfg(&u->u_instr_sequence); + if (g == NULL) { goto error; } - int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); + int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); assert(u->u_metadata.u_firstlineno); - if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals, + + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, nparams, u->u_metadata.u_firstlineno) < 0) { goto error; } - int stackdepth = _PyCfg_Stackdepth(&g); - if (stackdepth < 0) { + int stackdepth; + int nlocalsplus; + if (_PyCfg_OptimizedCfgToInstructionSequence(g, &u->u_metadata, code_flags, + &stackdepth, &nlocalsplus, + &optimized_instrs) < 0) { goto error; } - /* prepare_localsplus adds instructions for generators that push - * and pop an item on the stack. This assertion makes sure there - * is space on the stack for that. - * It should always be true, because a generator must have at - * least one expression or call to INTRINSIC_STOPITERATION_ERROR, - * which requires stackspace. - */ - assert(!(IS_GENERATOR(code_flags) && stackdepth == 0)); - /** Assembly **/ - int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags); - if (nlocalsplus < 0) { - goto error; - } - - _PyCfg_ConvertPseudoOps(g.g_entryblock); - - /* Order of basic blocks must have been determined by now */ - - if (_PyCfg_ResolveJumps(&g) < 0) { - goto error; - } - - /* Can't modify the bytecode after computing jump offsets. */ - - if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { - goto error; - } co = _PyAssemble_MakeCodeObject(&u->u_metadata, const_cache, consts, stackdepth, &optimized_instrs, nlocalsplus, @@ -7767,7 +7544,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, error: Py_XDECREF(consts); instr_sequence_fini(&optimized_instrs); - _PyCfgBuilder_Fini(&g); + _PyCfgBuilder_Free(g); return co; } @@ -7790,39 +7567,6 @@ optimize_and_assemble(struct compiler *c, int addNone) return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename); } -static int -cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq) -{ - int lbl = 0; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; - } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(instr_sequence_use_label(seq, b->b_label.id)); - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (OPCODE_HAS_JUMP(instr->i_opcode)) { - instr->i_oparg = instr->i_target->b_label.id; - } - RETURN_IF_ERROR( - instr_sequence_addop(seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); - - _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; - if (instr->i_except != NULL) { - hi->h_label = instr->i_except->b_label.id; - hi->h_startdepth = instr->i_except->b_startdepth; - hi->h_preserve_lasti = instr->i_except->b_preserve_lasti; - } - else { - hi->h_label = -1; - } - } - } - return SUCCESS; -} - - /* Access to compiler optimizations for unit tests. * * _PyCompile_CodeGen takes and AST, applies code-gen and @@ -7872,7 +7616,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) for (int i = 0; i < num_insts; i++) { if (is_target[i]) { - if (instr_sequence_use_label(seq, i) < 0) { + if (_PyCompile_InstructionSequence_UseLabel(seq, i) < 0) { goto error; } } @@ -7912,7 +7656,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) if (PyErr_Occurred()) { goto error; } - if (instr_sequence_addop(seq, opcode, oparg, loc) < 0) { + if (_PyCompile_InstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { goto error; } } @@ -7923,23 +7667,26 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) return ERROR; } -static int -instructions_to_cfg(PyObject *instructions, cfg_builder *g) +static cfg_builder* +instructions_to_cfg(PyObject *instructions) { + cfg_builder *g = NULL; instr_sequence seq; memset(&seq, 0, sizeof(instr_sequence)); if (instructions_to_instr_sequence(instructions, &seq) < 0) { goto error; } - if (instr_sequence_to_cfg(&seq, g) < 0) { + g = instr_sequence_to_cfg(&seq); + if (g == NULL) { goto error; } instr_sequence_fini(&seq); - return SUCCESS; + return g; error: + _PyCfgBuilder_Free(g); instr_sequence_fini(&seq); - return ERROR; + return NULL; } static PyObject * @@ -7978,42 +7725,14 @@ instr_sequence_to_instructions(instr_sequence *seq) static PyObject * cfg_to_instructions(cfg_builder *g) { - PyObject *instructions = PyList_New(0); - if (instructions == NULL) { + instr_sequence seq; + memset(&seq, 0, sizeof(seq)); + if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { return NULL; } - int lbl = 0; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; - } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - location loc = instr->i_loc; - int arg = HAS_TARGET(instr->i_opcode) ? - instr->i_target->b_label.id : instr->i_oparg; - - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, arg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); - if (inst_tuple == NULL) { - goto error; - } - - if (PyList_Append(instructions, inst_tuple) != 0) { - Py_DECREF(inst_tuple); - goto error; - } - Py_DECREF(inst_tuple); - } - } - - return instructions; -error: - Py_DECREF(instructions); - return NULL; + PyObject *res = instr_sequence_to_instructions(&seq); + instr_sequence_fini(&seq); + return res; } // C implementation of inspect.cleandoc() @@ -8195,34 +7914,36 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, PyObject * _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) { + cfg_builder *g = NULL; PyObject *res = NULL; PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; } - cfg_builder g; - if (instructions_to_cfg(instructions, &g) < 0) { + g = instructions_to_cfg(instructions); + if (g == NULL) { goto error; } int nparams = 0, firstlineno = 1; - if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals, + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, nparams, firstlineno) < 0) { goto error; } - res = cfg_to_instructions(&g); + res = cfg_to_instructions(g); error: Py_DECREF(const_cache); - _PyCfgBuilder_Fini(&g); + _PyCfgBuilder_Free(g); return res; } -int _PyCfg_JumpLabelsToTargets(basicblock *entryblock); +int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, PyObject *instructions) { + cfg_builder *g = NULL; PyCodeObject *co = NULL; instr_sequence optimized_instrs; memset(&optimized_instrs, 0, sizeof(instr_sequence)); @@ -8232,37 +7953,20 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - cfg_builder g; - if (instructions_to_cfg(instructions, &g) < 0) { + g = instructions_to_cfg(instructions); + if (g == NULL) { goto error; } - if (_PyCfg_JumpLabelsToTargets(g.g_entryblock) < 0) { - goto error; - } - - int stackdepth = _PyCfg_Stackdepth(&g); - if (stackdepth < 0) { + if (_PyCfg_JumpLabelsToTargets(g) < 0) { goto error; } int code_flags = 0; - int nlocalsplus = prepare_localsplus(umd, &g, code_flags); - if (nlocalsplus < 0) { - goto error; - } - - _PyCfg_ConvertPseudoOps(g.g_entryblock); - - /* Order of basic blocks must have been determined by now */ - - if (_PyCfg_ResolveJumps(&g) < 0) { - goto error; - } - - /* Can't modify the bytecode after computing jump offsets. */ - - if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { + int stackdepth, nlocalsplus; + if (_PyCfg_OptimizedCfgToInstructionSequence(g, umd, code_flags, + &stackdepth, &nlocalsplus, + &optimized_instrs) < 0) { goto error; } @@ -8277,7 +7981,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, error: Py_DECREF(const_cache); - _PyCfgBuilder_Fini(&g); + _PyCfgBuilder_Free(g); instr_sequence_fini(&optimized_instrs); return co; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 5b6b3f3cfefb03..9d7865661a8036 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -24,15 +24,75 @@ typedef _PyCompilerSrcLocation location; typedef _PyCfgJumpTargetLabel jump_target_label; -typedef _PyCfgBasicblock basicblock; -typedef _PyCfgBuilder cfg_builder; -typedef _PyCfgInstruction cfg_instr; + +typedef struct _PyCfgInstruction { + int i_opcode; + int i_oparg; + _PyCompilerSrcLocation i_loc; + struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */ + struct _PyCfgBasicblock *i_except; /* target block when exception is raised */ +} cfg_instr; + +typedef struct _PyCfgBasicblock { + /* Each basicblock in a compilation unit is linked via b_list in the + reverse order that the block are allocated. b_list points to the next + block in this list, not to be confused with b_next, which is next by + control flow. */ + struct _PyCfgBasicblock *b_list; + /* The label of this block if it is a jump target, -1 otherwise */ + _PyCfgJumpTargetLabel b_label; + /* Exception stack at start of block, used by assembler to create the exception handling table */ + struct _PyCfgExceptStack *b_exceptstack; + /* pointer to an array of instructions, initially NULL */ + cfg_instr *b_instr; + /* If b_next is non-NULL, it is a pointer to the next + block reached by normal control flow. */ + struct _PyCfgBasicblock *b_next; + /* number of instructions used */ + int b_iused; + /* length of instruction array (b_instr) */ + int b_ialloc; + /* Used by add_checks_for_loads_of_unknown_variables */ + uint64_t b_unsafe_locals_mask; + /* Number of predecessors that a block has. */ + int b_predecessors; + /* depth of stack upon entry of block, computed by stackdepth() */ + int b_startdepth; + /* Basic block is an exception handler that preserves lasti */ + unsigned b_preserve_lasti : 1; + /* Used by compiler passes to mark whether they have visited a basic block. */ + unsigned b_visited : 1; + /* b_except_handler is used by the cold-detection algorithm to mark exception targets */ + unsigned b_except_handler : 1; + /* b_cold is true if this block is not perf critical (like an exception handler) */ + unsigned b_cold : 1; + /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */ + unsigned b_warm : 1; +} basicblock; + + +struct _PyCfgBuilder { + /* The entryblock, at which control flow begins. All blocks of the + CFG are reachable through the b_next links */ + struct _PyCfgBasicblock *g_entryblock; + /* Pointer to the most recently allocated block. By following + b_list links, you can reach all allocated blocks. */ + struct _PyCfgBasicblock *g_block_list; + /* pointer to the block currently being constructed */ + struct _PyCfgBasicblock *g_curblock; + /* label for the next instruction to be placed */ + _PyCfgJumpTargetLabel g_current_label; +}; + +typedef struct _PyCfgBuilder cfg_builder; static const jump_target_label NO_LABEL = {-1}; #define SAME_LABEL(L1, L2) ((L1).id == (L2).id) #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) +#define LOCATION(LNO, END_LNO, COL, END_COL) \ + ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) static inline int is_block_push(cfg_instr *i) @@ -50,7 +110,7 @@ is_jump(cfg_instr *i) #define INSTR_SET_OP1(I, OP, ARG) \ do { \ assert(OPCODE_HAS_ARG(OP)); \ - _PyCfgInstruction *_instr__ptr_ = (I); \ + cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = (ARG); \ } while (0); @@ -59,7 +119,7 @@ is_jump(cfg_instr *i) #define INSTR_SET_OP0(I, OP) \ do { \ assert(!OPCODE_HAS_ARG(OP)); \ - _PyCfgInstruction *_instr__ptr_ = (I); \ + cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = 0; \ } while (0); @@ -148,8 +208,8 @@ basicblock_last_instr(const basicblock *b) { } static inline int -basicblock_nofallthrough(const _PyCfgBasicblock *b) { - _PyCfgInstruction *last = basicblock_last_instr(b); +basicblock_nofallthrough(const basicblock *b) { + cfg_instr *last = basicblock_last_instr(b); return (last && (IS_SCOPE_EXIT_OPCODE(last->i_opcode) || IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode))); @@ -175,8 +235,8 @@ copy_basicblock(cfg_builder *g, basicblock *block) return result; } -int -_PyBasicblock_InsertInstruction(basicblock *block, int pos, cfg_instr *instr) { +static int +basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) { RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; @@ -311,8 +371,8 @@ cfg_builder_check(cfg_builder *g) } #endif -int -_PyCfgBuilder_Init(cfg_builder *g) +static int +init_cfg_builder(cfg_builder *g) { g->g_block_list = NULL; basicblock *block = cfg_builder_new_block(g); @@ -324,9 +384,28 @@ _PyCfgBuilder_Init(cfg_builder *g) return SUCCESS; } +cfg_builder * +_PyCfgBuilder_New(void) +{ + cfg_builder *g = PyMem_Malloc(sizeof(cfg_builder)); + if (g == NULL) { + PyErr_NoMemory(); + return NULL; + } + memset(g, 0, sizeof(cfg_builder)); + if (init_cfg_builder(g) < 0) { + PyMem_Free(g); + return NULL; + } + return g; +} + void -_PyCfgBuilder_Fini(cfg_builder* g) +_PyCfgBuilder_Free(cfg_builder *g) { + if (g == NULL) { + return; + } assert(cfg_builder_check(g)); basicblock *b = g->g_block_list; while (b != NULL) { @@ -337,6 +416,21 @@ _PyCfgBuilder_Fini(cfg_builder* g) PyObject_Free((void *)b); b = next; } + PyMem_Free(g); +} + +int +_PyCfgBuilder_CheckSize(cfg_builder *g) +{ + int nblocks = 0; + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { + nblocks++; + } + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { + PyErr_NoMemory(); + return ERROR; + } + return SUCCESS; } int @@ -450,7 +544,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { static int -normalize_jumps(_PyCfgBuilder *g) +normalize_jumps(cfg_builder *g) { basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { @@ -463,14 +557,6 @@ normalize_jumps(_PyCfgBuilder *g) return SUCCESS; } -int -_PyCfg_ResolveJumps(_PyCfgBuilder *g) -{ - RETURN_IF_ERROR(normalize_jumps(g)); - assert(no_redundant_jumps(g)); - return SUCCESS; -} - static int check_cfg(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -529,9 +615,9 @@ translate_jump_labels_to_targets(basicblock *entryblock) } int -_PyCfg_JumpLabelsToTargets(basicblock *entryblock) +_PyCfg_JumpLabelsToTargets(cfg_builder *g) { - return translate_jump_labels_to_targets(entryblock); + return translate_jump_labels_to_targets(g->g_entryblock); } static int @@ -553,10 +639,14 @@ mark_except_handlers(basicblock *entryblock) { } -typedef _PyCfgExceptStack ExceptStack; +struct _PyCfgExceptStack { + basicblock *handlers[CO_MAXBLOCKS+1]; + int depth; +}; + static basicblock * -push_except_block(ExceptStack *stack, cfg_instr *setup) { +push_except_block(struct _PyCfgExceptStack *stack, cfg_instr *setup) { assert(is_block_push(setup)); int opcode = setup->i_opcode; basicblock * target = setup->i_target; @@ -568,19 +658,19 @@ push_except_block(ExceptStack *stack, cfg_instr *setup) { } static basicblock * -pop_except_block(ExceptStack *stack) { +pop_except_block(struct _PyCfgExceptStack *stack) { assert(stack->depth > 0); return stack->handlers[--stack->depth]; } static basicblock * -except_stack_top(ExceptStack *stack) { +except_stack_top(struct _PyCfgExceptStack *stack) { return stack->handlers[stack->depth]; } -static ExceptStack * +static struct _PyCfgExceptStack * make_except_stack(void) { - ExceptStack *new = PyMem_Malloc(sizeof(ExceptStack)); + struct _PyCfgExceptStack *new = PyMem_Malloc(sizeof(struct _PyCfgExceptStack)); if (new == NULL) { PyErr_NoMemory(); return NULL; @@ -590,14 +680,14 @@ make_except_stack(void) { return new; } -static ExceptStack * -copy_except_stack(ExceptStack *stack) { - ExceptStack *copy = PyMem_Malloc(sizeof(ExceptStack)); +static struct _PyCfgExceptStack * +copy_except_stack(struct _PyCfgExceptStack *stack) { + struct _PyCfgExceptStack *copy = PyMem_Malloc(sizeof(struct _PyCfgExceptStack)); if (copy == NULL) { PyErr_NoMemory(); return NULL; } - memcpy(copy, stack, sizeof(ExceptStack)); + memcpy(copy, stack, sizeof(struct _PyCfgExceptStack)); return copy; } @@ -633,8 +723,8 @@ stackdepth_push(basicblock ***sp, basicblock *b, int depth) /* Find the flow path that needs the largest stack. We assume that * cycles in the flow graph have no net effect on the stack depth. */ -int -_PyCfg_Stackdepth(cfg_builder *g) +static int +calculate_stackdepth(cfg_builder *g) { basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { @@ -723,7 +813,7 @@ label_exception_targets(basicblock *entryblock) { if (todo_stack == NULL) { return ERROR; } - ExceptStack *except_stack = make_except_stack(); + struct _PyCfgExceptStack *except_stack = make_except_stack(); if (except_stack == NULL) { PyMem_Free(todo_stack); PyErr_NoMemory(); @@ -747,7 +837,7 @@ label_exception_targets(basicblock *entryblock) { cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { if (!instr->i_target->b_visited) { - ExceptStack *copy = copy_except_stack(except_stack); + struct _PyCfgExceptStack *copy = copy_except_stack(except_stack); if (copy == NULL) { goto error; } @@ -766,7 +856,7 @@ label_exception_targets(basicblock *entryblock) { assert(i == b->b_iused -1); if (!instr->i_target->b_visited) { if (BB_HAS_FALLTHROUGH(b)) { - ExceptStack *copy = copy_except_stack(except_stack); + struct _PyCfgExceptStack *copy = copy_except_stack(except_stack); if (copy == NULL) { goto error; } @@ -2103,8 +2193,8 @@ push_cold_blocks_to_end(cfg_builder *g) { return SUCCESS; } -void -_PyCfg_ConvertPseudoOps(basicblock *entryblock) +static void +convert_pseudo_ops(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { @@ -2293,3 +2383,270 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); return SUCCESS; } + +static int * +build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) +{ + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); + + int noffsets = ncellvars + nfreevars; + int *fixed = PyMem_New(int, noffsets); + if (fixed == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (int i = 0; i < noffsets; i++) { + fixed[i] = nlocals + i; + } + + PyObject *varname, *cellindex; + Py_ssize_t pos = 0; + while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); + if (varindex != NULL) { + assert(PyLong_AS_LONG(cellindex) < INT_MAX); + assert(PyLong_AS_LONG(varindex) < INT_MAX); + int oldindex = (int)PyLong_AS_LONG(cellindex); + int argoffset = (int)PyLong_AS_LONG(varindex); + fixed[oldindex] = argoffset; + } + } + + return fixed; +} + +#define IS_GENERATOR(CF) \ + ((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) + +static int +insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, + int *fixed, int nfreevars, int code_flags) +{ + assert(umd->u_firstlineno > 0); + + /* Add the generator prefix instructions. */ + if (IS_GENERATOR(code_flags)) { + /* Note that RETURN_GENERATOR + POP_TOP have a net stack effect + * of 0. This is because RETURN_GENERATOR pushes an element + * with _PyFrame_StackPush before switching stacks. + */ + cfg_instr make_gen = { + .i_opcode = RETURN_GENERATOR, + .i_oparg = 0, + .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); + cfg_instr pop_top = { + .i_opcode = POP_TOP, + .i_oparg = 0, + .i_loc = NO_LOCATION, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); + } + + /* Set up cells for any variable that escapes, to be put in a closure. */ + const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + if (ncellvars) { + // umd->u_cellvars has the cells out of order so we sort them + // before adding the MAKE_CELL instructions. Note that we + // adjust for arg cells, which come first. + const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); + int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); + if (sorted == NULL) { + PyErr_NoMemory(); + return ERROR; + } + for (int i = 0; i < ncellvars; i++) { + sorted[fixed[i]] = i + 1; + } + for (int i = 0, ncellsused = 0; ncellsused < ncellvars; i++) { + int oldindex = sorted[i] - 1; + if (oldindex == -1) { + continue; + } + cfg_instr make_cell = { + .i_opcode = MAKE_CELL, + // This will get fixed in offset_derefs(). + .i_oparg = oldindex, + .i_loc = NO_LOCATION, + .i_target = NULL, + }; + if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) { + PyMem_RawFree(sorted); + return ERROR; + } + ncellsused += 1; + } + PyMem_RawFree(sorted); + } + + if (nfreevars) { + cfg_instr copy_frees = { + .i_opcode = COPY_FREE_VARS, + .i_oparg = nfreevars, + .i_loc = NO_LOCATION, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, ©_frees)); + } + + return SUCCESS; +} + +static int +fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) +{ + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); + int noffsets = ncellvars + nfreevars; + + // First deal with duplicates (arg cells). + int numdropped = 0; + for (int i = 0; i < noffsets ; i++) { + if (fixedmap[i] == i + nlocals) { + fixedmap[i] -= numdropped; + } + else { + // It was a duplicate (cell/arg). + numdropped += 1; + } + } + + // Then update offsets, either relative to locals or by cell2arg. + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *inst = &b->b_instr[i]; + // This is called before extended args are generated. + assert(inst->i_opcode != EXTENDED_ARG); + int oldoffset = inst->i_oparg; + switch(inst->i_opcode) { + case MAKE_CELL: + case LOAD_CLOSURE: + case LOAD_DEREF: + case STORE_DEREF: + case DELETE_DEREF: + case LOAD_FROM_DICT_OR_DEREF: + assert(oldoffset >= 0); + assert(oldoffset < noffsets); + assert(fixedmap[oldoffset] >= 0); + inst->i_oparg = fixedmap[oldoffset]; + } + } + } + + return numdropped; +} + +static int +prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) +{ + assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); + assert(INT_MAX - nlocals - ncellvars > 0); + assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); + int nlocalsplus = nlocals + ncellvars + nfreevars; + int* cellfixedoffsets = build_cellfixedoffsets(umd); + if (cellfixedoffsets == NULL) { + return ERROR; + } + + // This must be called before fix_cell_offsets(). + if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + PyMem_Free(cellfixedoffsets); + return ERROR; + } + + int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); + PyMem_Free(cellfixedoffsets); // At this point we're done with it. + cellfixedoffsets = NULL; + if (numdropped < 0) { + return ERROR; + } + + nlocalsplus -= numdropped; + return nlocalsplus; +} + +int +_PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq) +{ + int lbl = 0; + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + b->b_label = (jump_target_label){lbl}; + lbl += b->b_iused; + } + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (OPCODE_HAS_JUMP(instr->i_opcode)) { + instr->i_oparg = instr->i_target->b_label.id; + } + RETURN_IF_ERROR( + _PyCompile_InstructionSequence_Addop( + seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); + + _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; + if (instr->i_except != NULL) { + hi->h_label = instr->i_except->b_label.id; + hi->h_startdepth = instr->i_except->b_startdepth; + hi->h_preserve_lasti = instr->i_except->b_preserve_lasti; + } + else { + hi->h_label = -1; + } + } + } + return SUCCESS; +} + + +int +_PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, + _PyCompile_CodeUnitMetadata *umd, int code_flags, + int *stackdepth, int *nlocalsplus, + _PyCompile_InstructionSequence *seq) +{ + *stackdepth = calculate_stackdepth(g); + if (*stackdepth < 0) { + return ERROR; + } + + /* prepare_localsplus adds instructions for generators that push + * and pop an item on the stack. This assertion makes sure there + * is space on the stack for that. + * It should always be true, because a generator must have at + * least one expression or call to INTRINSIC_STOPITERATION_ERROR, + * which requires stackspace. + */ + assert(!(IS_GENERATOR(code_flags) && *stackdepth == 0)); + + *nlocalsplus = prepare_localsplus(umd, g, code_flags); + if (*nlocalsplus < 0) { + return ERROR; + } + + convert_pseudo_ops(g->g_entryblock); + + /* Order of basic blocks must have been determined by now */ + + RETURN_IF_ERROR(normalize_jumps(g)); + assert(no_redundant_jumps(g)); + + /* Can't modify the bytecode after computing jump offsets. */ + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { + return ERROR; + } + + return SUCCESS; +} + From 1d976b2da26cd62b5d0e860e3055df6fb893577f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Aug 2023 13:34:00 +0100 Subject: [PATCH 101/598] GH-106485: Handle dict subclasses correctly when dematerializing `__dict__` (GH-107837) --- Include/internal/pycore_dict.h | 2 +- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + Lib/test/test_opcache.py | 121 ++++++++++++++++++ Modules/_testinternalcapi.c | 36 ++++++ Objects/dictobject.c | 14 +- 7 files changed, 169 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6ae81c033c43aa..2ad6ef0f7c04d5 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -199,7 +199,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -extern int _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); extern PyObject *_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6d50ffd0a02f1f..ee9010583ff8b5 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -547,6 +547,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_listcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_module)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index bb1fb13f342fc6..b081c0e023fa4a 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -33,6 +33,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(anon_lambda, "") STRUCT_FOR_STR(anon_listcomp, "") STRUCT_FOR_STR(anon_module, "") + STRUCT_FOR_STR(anon_null, "") STRUCT_FOR_STR(anon_setcomp, "") STRUCT_FOR_STR(anon_string, "") STRUCT_FOR_STR(anon_unknown, "") diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 2d66647438b193..8c9c7f753d8579 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -539,6 +539,7 @@ extern "C" { INIT_STR(anon_lambda, ""), \ INIT_STR(anon_listcomp, ""), \ INIT_STR(anon_module, ""), \ + INIT_STR(anon_null, ""), \ INIT_STR(anon_setcomp, ""), \ INIT_STR(anon_string, ""), \ INIT_STR(anon_unknown, ""), \ diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7d317c012a304e..1baa10cbdfdef2 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1,8 +1,11 @@ +import copy +import pickle import dis import threading import types import unittest from test.support import threading_helper +import _testinternalcapi class TestLoadSuperAttrCache(unittest.TestCase): @@ -997,6 +1000,124 @@ def write(items): opname = "UNPACK_SEQUENCE_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) +class C: + pass + +class TestInstanceDict(unittest.TestCase): + + def setUp(self): + c = C() + c.a, c.b, c.c = 0,0,0 + + def test_values_on_instance(self): + c = C() + c.a = 1 + C().b = 2 + c.c = 3 + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, '', 3) + ) + + def test_dict_materialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + + def test_dict_dematerialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + for _ in range(100): + c.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + + def test_dict_dematerialization_multiple_refs(self): + c = C() + c.a = 1 + c.b = 2 + d = c.__dict__ + for _ in range(100): + c.a + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + self.assertIs(c.__dict__, d) + + def test_dict_dematerialization_copy(self): + c = C() + c.a = 1 + c.b = 2 + c2 = copy.copy(c) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '') + ) + c3 = copy.deepcopy(c) + for _ in range(100): + c.a + c3.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + #NOTE -- c3.__dict__ does not de-materialize + + def test_dict_dematerialization_pickle(self): + c = C() + c.a = 1 + c.b = 2 + c2 = pickle.loads(pickle.dumps(c)) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '') + ) + + def test_dict_dematerialization_subclass(self): + class D(dict): pass + c = C() + c.a = 1 + c.b = 2 + c.__dict__ = D(c.__dict__) + for _ in range(100): + c.a + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + self.assertEqual( + c.__dict__, + {'a':1, 'b':2} + ) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f28b46dd9846c8..d1082c7dae8aee 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -15,6 +15,7 @@ #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc #include "pycore_ceval.h" // _PyEval_AddPendingCall +#include "pycore_dict.h" // _PyDictOrValues_GetValues #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -1530,6 +1531,40 @@ test_pymem_getallocatorsname(PyObject *self, PyObject *args) return PyUnicode_FromString(name); } +static PyObject * +get_object_dict_values(PyObject *self, PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + Py_RETURN_NONE; + } + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); + if (!_PyDictOrValues_IsValues(dorv)) { + Py_RETURN_NONE; + } + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(keys != NULL); + int size = (int)keys->dk_nentries; + assert(size >= 0); + PyObject *res = PyTuple_New(size); + if (res == NULL) { + return NULL; + } + _Py_DECLARE_STR(anon_null, ""); + for(int i = 0; i < size; i++) { + PyObject *item = values->values[i]; + if (item == NULL) { + item = &_Py_STR(anon_null); + } + else { + Py_INCREF(item); + } + PyTuple_SET_ITEM(res, i, item); + } + return res; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, @@ -1594,6 +1629,7 @@ static PyMethodDef module_functions[] = { {"check_pyobject_uninitialized_is_freed", check_pyobject_uninitialized_is_freed, METH_NOARGS}, {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, + {"get_object_dict_values", get_object_dict_values, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 931103cdc1a064..36375f50646fd4 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5464,22 +5464,24 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } -// Return 1 if the dict was dematerialized, 0 otherwise. -int +// Return true if the dict was dematerialized, false otherwise. +bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) { assert(_PyObject_DictOrValuesPointer(obj) == dorv); assert(!_PyDictOrValues_IsValues(*dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); if (dict == NULL) { - return 0; + return false; } // It's likely that this dict still shares its keys (if it was materialized // on request and not heavily modified): - assert(PyDict_CheckExact(dict)); + if (!PyDict_CheckExact(dict)) { + return false; + } assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { - return 0; + return false; } assert(dict->ma_values); // We have an opportunity to do something *really* cool: dematerialize it! @@ -5490,7 +5492,7 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) dict->ma_keys = NULL; dict->ma_values = NULL; Py_DECREF(dict); - return 1; + return true; } int From 37d8b904f8b5b660f556597b21c0933b841d18de Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Aug 2023 13:35:02 +0100 Subject: [PATCH 102/598] GH-107674: Avoid allocating boxed ints for `sys.settrace` line events (GH-107780) --- ...-08-05-04-47-18.gh-issue-107674.0sYhR2.rst | 1 + Python/instrumentation.c | 44 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst new file mode 100644 index 00000000000000..acfbf1fa2adf2c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst @@ -0,0 +1 @@ +Fixed performance regression in ``sys.settrace``. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 64684ad522f687..6d11649c07fbe4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -961,7 +961,7 @@ call_instrumentation_vector( /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); - PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); + PyObject *offset_obj = PyLong_FromLong(bytes_offset); if (offset_obj == NULL) { return -1; } @@ -1141,14 +1141,46 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE] ); - PyObject *line_obj = PyLong_FromSsize_t(line); + /* Special case sys.settrace to avoid boxing the line number, + * only to immediately unbox it. */ + if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) { + if (tstate->c_tracefunc != NULL && line >= 0) { + PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame); + if (frame_obj == NULL) { + return -1; + } + if (frame_obj->f_trace_lines) { + /* Need to set tracing and what_event as if using + * the instrumentation call. */ + int old_what = tstate->what_event; + tstate->what_event = PY_MONITORING_EVENT_LINE; + tstate->tracing++; + /* Call c_tracefunc directly, having set the line number. */ + Py_INCREF(frame_obj); + frame_obj->f_lineno = line; + int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None); + frame_obj->f_lineno = 0; + tstate->tracing--; + tstate->what_event = old_what; + Py_DECREF(frame_obj); + if (err) { + return -1; + } + } + } + tools &= (255 - (1 << PY_MONITORING_SYS_TRACE_ID)); + } + if (tools == 0) { + goto done; + } + PyObject *line_obj = PyLong_FromLong(line); if (line_obj == NULL) { return -1; } PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; - while (tools) { + do { int tool = most_significant_bit(tools); - assert(tool >= 0 && tool < 8); + assert(tool >= 0 && tool < PY_MONITORING_SYS_PROFILE_ID); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, &args[1], @@ -1166,7 +1198,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, /* DISABLE */ remove_line_tools(code, i, 1 << tool); } - } + } while (tools); Py_DECREF(line_obj); done: assert(original_opcode != 0); @@ -1195,7 +1227,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] ); int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); - PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); + PyObject *offset_obj = PyLong_FromLong(bytes_offset); if (offset_obj == NULL) { return -1; } From e4275f4df36a7cdd58cd4daa7d65b1947a2593d3 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Thu, 10 Aug 2023 09:39:13 -0500 Subject: [PATCH 103/598] gh-107838: In dataclasses, improve error message when a non-default field follows a default field. (gh-107842) Add the name of the previous default argument field in an error message. --- Lib/dataclasses.py | 6 +++--- Lib/test/test_dataclasses.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3c72c289e43882..21f3fa5c213f1f 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -575,15 +575,15 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, # message, and future-proofs us in case we build up the function # using ast. - seen_default = False + seen_default = None for f in std_fields: # Only consider the non-kw-only fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): - seen_default = True + seen_default = f elif seen_default: raise TypeError(f'non-default argument {f.name!r} ' - 'follows default argument') + f'follows default argument {seen_default.name!r}') locals = {f'__dataclass_type_{f.name}__': f.type for f in fields} locals.update({ diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 6669f1c57e2e78..bd8d82438414e6 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -134,7 +134,7 @@ class C: # Non-defaults following defaults. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class C: x: int = 0 @@ -143,7 +143,7 @@ class C: # A derived class adds a non-default field after a default one. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class B: x: int = 0 @@ -156,7 +156,7 @@ class C(B): # a field which didn't use to have a default. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class B: x: int @@ -4521,7 +4521,7 @@ class A: # Make sure we still check for non-kwarg non-defaults not following # defaults. - err_regex = "non-default argument 'z' follows default argument" + err_regex = "non-default argument 'z' follows default argument 'a'" with self.assertRaisesRegex(TypeError, err_regex): @dataclass class A: From 16dcce21768ba381996a88ac8c255bf1490b3680 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 10 Aug 2023 17:55:47 +0200 Subject: [PATCH 104/598] gh-107810: Improve DeprecationWarning for metaclasses with custom tp_new (GH-107834) Co-authored-by: Kirill Podoprigora --- Lib/test/test_capi/test_misc.py | 2 +- .../C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst | 1 + Objects/typeobject.c | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 001d37de8e0eb3..c81212202d9ef2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -611,7 +611,7 @@ class Base(metaclass=metaclass): # Class creation from C with warnings_helper.check_warnings( - ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), + ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), ): sub = _testcapi.make_type_with_base(Base) self.assertTrue(issubclass(sub, Base)) diff --git a/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst b/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst new file mode 100644 index 00000000000000..c8a1f6d122b61b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst @@ -0,0 +1 @@ +Improve :exc:`DeprecationWarning` for uses of :c:type:`PyType_Spec` with metaclasses that have custom ``tp_new``. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 71e96f5af4e419..aca14e79a19979 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4264,9 +4264,9 @@ _PyType_FromMetaclass_impl( if (_allow_tp_new) { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Using PyType_Spec with metaclasses that have custom " - "tp_new is deprecated and will no longer be allowed in " - "Python 3.14.") < 0) { + "Type %s uses PyType_Spec with a metaclass that has custom " + "tp_new. This is deprecated and will no longer be allowed in " + "Python 3.14.", spec->name) < 0) { goto finally; } } From 50bbc56009ae7303d2482f28eb62f2603664b58f Mon Sep 17 00:00:00 2001 From: Martin DeMello Date: Thu, 10 Aug 2023 13:43:14 -0700 Subject: [PATCH 105/598] Fix the long64 reader in umarshal.py (GH-107828) --- Tools/build/umarshal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index f61570cbaff751..e05d93cf23c921 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -125,10 +125,10 @@ def r_long64(self) -> int: x |= buf[1] << 8 x |= buf[2] << 16 x |= buf[3] << 24 - x |= buf[1] << 32 - x |= buf[1] << 40 - x |= buf[1] << 48 - x |= buf[1] << 56 + x |= buf[4] << 32 + x |= buf[5] << 40 + x |= buf[6] << 48 + x |= buf[7] << 56 x |= -(x & (1<<63)) # Sign-extend return x From 23a6db98f21cba3af69a921f01613bd5f602bf6d Mon Sep 17 00:00:00 2001 From: shailshouryya <42100758+shailshouryya@users.noreply.github.com> Date: Thu, 10 Aug 2023 21:43:13 -0700 Subject: [PATCH 106/598] gh-107421: Clarify `OrderedDict` Examples and Recipes (#107613) --- Doc/library/collections.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index bb46782c06e1c8..b8b231bb15b1b0 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1224,7 +1224,7 @@ variants of :func:`functools.lru_cache`: result = self.func(*args) self.cache[args] = time(), result if len(self.cache) > self.maxsize: - self.cache.popitem(0) + self.cache.popitem(last=False) return result @@ -1256,12 +1256,12 @@ variants of :func:`functools.lru_cache`: if self.requests[args] <= self.cache_after: self.requests.move_to_end(args) if len(self.requests) > self.maxrequests: - self.requests.popitem(0) + self.requests.popitem(last=False) else: self.requests.pop(args, None) self.cache[args] = result if len(self.cache) > self.maxsize: - self.cache.popitem(0) + self.cache.popitem(last=False) return result .. doctest:: From 3901c991e169da6fba8c0033a86a6f2e6146bb7f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 11 Aug 2023 18:08:38 +0300 Subject: [PATCH 107/598] gh-84805: Autogenerate signature for METH_NOARGS and METH_O extension functions (GH-107794) --- Include/internal/pycore_object.h | 2 +- Lib/test/test_inspect.py | 77 +++++++++++++++++++ Lib/test/test_pydoc.py | 50 ++++++++++++ Lib/test/test_rlcompleter.py | 5 +- ...3-08-09-08-31-20.gh-issue-84805.7JRWua.rst | 2 + Modules/_testcapi/docstring.c | 64 ++++++++++++--- Objects/descrobject.c | 10 ++- Objects/methodobject.c | 4 +- Objects/typeobject.c | 29 ++++++- Tools/c-analyzer/cpython/ignored.tsv | 1 + 10 files changed, 227 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7cdf64bcdacb9f..857d6efec3b3b1 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -381,7 +381,7 @@ extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); -extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); +extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); extern int _PyObject_InitializeDict(PyObject *obj); int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5c31748ce2caac..07c48eac5b48b0 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -13,7 +13,9 @@ import _pickle import pickle import shutil +import stat import sys +import time import types import tempfile import textwrap @@ -22,6 +24,7 @@ import unittest.mock import warnings + try: from concurrent.futures import ThreadPoolExecutor except ImportError: @@ -136,6 +139,14 @@ def gen_coroutine_function_example(self): yield return 'spam' +def meth_noargs(): pass +def meth_o(object, /): pass +def meth_self_noargs(self, /): pass +def meth_self_o(self, object, /): pass +def meth_type_noargs(type, /): pass +def meth_type_o(type, object, /): pass + + class TestPredicates(IsTestBase): def test_excluding_predicates(self): @@ -1173,6 +1184,39 @@ def test_getfullargspec_builtin_func_no_signature(self): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + for builtin, template in [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_self_noargs), + (obj.meth_o, meth_self_o), + (cls.meth_noargs_class, meth_type_noargs), + (cls.meth_o_class, meth_type_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (stat.S_IMODE, meth_o), + (str.lower, meth_self_noargs), + (''.lower, meth_self_noargs), + (set.add, meth_self_o), + (set().add, meth_self_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_self_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_type_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_type_o), + ]: + with self.subTest(builtin): + self.assertEqual(inspect.getfullargspec(builtin), + inspect.getfullargspec(template)) + def test_getfullargspec_definition_order_preserved_on_kwonly(self): for fn in signatures_with_lexicographic_keyword_only_parameters(): signature = inspect.getfullargspec(fn) @@ -2888,6 +2932,39 @@ def test_signature_on_builtins_no_signature(self): 'no signature found for builtin'): inspect.signature(str) + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + for builtin, template in [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_noargs), + (obj.meth_o, meth_o), + (cls.meth_noargs_class, meth_noargs), + (cls.meth_o_class, meth_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (stat.S_IMODE, meth_o), + (str.lower, meth_self_noargs), + (''.lower, meth_noargs), + (set.add, meth_self_o), + (set().add, meth_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_o), + ]: + with self.subTest(builtin): + self.assertEqual(inspect.signature(builtin), + inspect.signature(template)) + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ddb5187f90da9b..8df8b608cf959d 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1,3 +1,4 @@ +import datetime import os import sys import contextlib @@ -12,6 +13,7 @@ import stat import tempfile import test.support +import time import types import typing import unittest @@ -1180,6 +1182,54 @@ def test_module_level_callable(self): self.assertEqual(self._get_summary_line(os.stat), "stat(path, *, dir_fd=None, follow_symlinks=True)") + def test_module_level_callable_noargs(self): + self.assertEqual(self._get_summary_line(time.time), + "time()") + + def test_module_level_callable_o(self): + self.assertEqual(self._get_summary_line(stat.S_IMODE), + "S_IMODE(object, /)") + + def test_unbound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(str.lower), + "lower(self, /)") + + def test_bound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(''.lower), + "lower() method of builtins.str instance") + + def test_unbound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set.add), + "add(self, object, /)") + + def test_bound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set().add), + "add(object, /) method of builtins.set instance") + + def test_unbound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set.__contains__), + "__contains__(self, object, /)") + + def test_bound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set().__contains__), + "__contains__(object, /) method of builtins.set instance") + + def test_unbound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), + "utcnow(type, /)") + + def test_bound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), + "utcnow() method of builtins.type instance") + + def test_unbound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), + "__class_getitem__(type, object, /)") + + def test_bound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__class_getitem__), + "__class_getitem__(object, /) method of builtins.type instance") + @requires_docstrings def test_staticmethod(self): class X: diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 6b5fc9a0247f4b..7347fca71be2fe 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -53,7 +53,10 @@ def test_attr_matches(self): ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) - expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') + expected = sorted({'None.%s%s' % (x, + '()' if x == '__init_subclass__' + else '' if x == '__doc__' + else '(') for x in dir(None)}) self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst new file mode 100644 index 00000000000000..23dfba989fa552 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst @@ -0,0 +1,2 @@ +Autogenerate signature for :c:macro:`METH_NOARGS` and :c:macro:`METH_O` +extension functions. diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index a997c54a8a69c4..b680171cc1437a 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -66,42 +66,88 @@ test_with_docstring(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef test_methods[] = { {"docstring_empty", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_empty}, {"docstring_no_signature", + (PyCFunction)test_with_docstring, METH_VARARGS, + docstring_no_signature}, + {"docstring_no_signature_noargs", (PyCFunction)test_with_docstring, METH_NOARGS, docstring_no_signature}, + {"docstring_no_signature_o", + (PyCFunction)test_with_docstring, METH_O, + docstring_no_signature}, {"docstring_with_invalid_signature", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_invalid_signature}, {"docstring_with_invalid_signature2", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_invalid_signature2}, {"docstring_with_signature", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature}, {"docstring_with_signature_and_extra_newlines", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature_and_extra_newlines}, {"docstring_with_signature_but_no_doc", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature_but_no_doc}, {"docstring_with_signature_with_defaults", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature_with_defaults}, {"no_docstring", - (PyCFunction)test_with_docstring, METH_NOARGS}, + (PyCFunction)test_with_docstring, METH_VARARGS}, {"test_with_docstring", - test_with_docstring, METH_NOARGS, + test_with_docstring, METH_VARARGS, PyDoc_STR("This is a pretty normal docstring.")}, {NULL}, }; +static PyMethodDef DocStringNoSignatureTest_methods[] = { + {"meth_noargs", + (PyCFunction)test_with_docstring, METH_NOARGS, + docstring_no_signature}, + {"meth_o", + (PyCFunction)test_with_docstring, METH_O, + docstring_no_signature}, + {"meth_noargs_class", + (PyCFunction)test_with_docstring, METH_NOARGS|METH_CLASS, + docstring_no_signature}, + {"meth_o_class", + (PyCFunction)test_with_docstring, METH_O|METH_CLASS, + docstring_no_signature}, + {"meth_noargs_static", + (PyCFunction)test_with_docstring, METH_NOARGS|METH_STATIC, + docstring_no_signature}, + {"meth_o_static", + (PyCFunction)test_with_docstring, METH_O|METH_STATIC, + docstring_no_signature}, + {"meth_noargs_coexist", + (PyCFunction)test_with_docstring, METH_NOARGS|METH_COEXIST, + docstring_no_signature}, + {"meth_o_coexist", + (PyCFunction)test_with_docstring, METH_O|METH_COEXIST, + docstring_no_signature}, + {NULL}, +}; + +static PyTypeObject DocStringNoSignatureTest = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testcapi.DocStringNoSignatureTest", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = DocStringNoSignatureTest_methods, + .tp_new = PyType_GenericNew, +}; + int _PyTestCapi_Init_Docstring(PyObject *mod) { if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } + if (PyModule_AddType(mod, &DocStringNoSignatureTest) < 0) { + return -1; + } return 0; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 60383dd06d1add..a744c3d1e58658 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -588,7 +588,9 @@ method_get_doc(PyMethodDescrObject *descr, void *closure) static PyObject * method_get_text_signature(PyMethodDescrObject *descr, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); + return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, + descr->d_method->ml_doc, + descr->d_method->ml_flags); } static PyObject * @@ -691,7 +693,8 @@ wrapperdescr_get_doc(PyWrapperDescrObject *descr, void *closure) static PyObject * wrapperdescr_get_text_signature(PyWrapperDescrObject *descr, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, descr->d_base->doc); + return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, + descr->d_base->doc, 0); } static PyGetSetDef wrapperdescr_getset[] = { @@ -1384,7 +1387,8 @@ wrapper_doc(wrapperobject *wp, void *Py_UNUSED(ignored)) static PyObject * wrapper_text_signature(wrapperobject *wp, void *Py_UNUSED(ignored)) { - return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc); + return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, + wp->descr->d_base->doc, 0); } static PyObject * diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 628d227ef33c39..521c9059770acb 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -192,7 +192,9 @@ static PyMethodDef meth_methods[] = { static PyObject * meth_get__text_signature__(PyCFunctionObject *m, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); + return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, + m->m_ml->ml_doc, + m->m_ml->ml_flags); } static PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index aca14e79a19979..030e8bfc99b6d7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -586,8 +586,29 @@ _PyType_GetDocFromInternalDoc(const char *name, const char *internal_doc) return PyUnicode_FromString(doc); } +static const char * +signature_from_flags(int flags) +{ + switch (flags & ~METH_COEXIST) { + case METH_NOARGS: + return "($self, /)"; + case METH_NOARGS|METH_CLASS: + return "($type, /)"; + case METH_NOARGS|METH_STATIC: + return "()"; + case METH_O: + return "($self, object, /)"; + case METH_O|METH_CLASS: + return "($type, object, /)"; + case METH_O|METH_STATIC: + return "(object, /)"; + default: + return NULL; + } +} + PyObject * -_PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_doc) +_PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_doc, int flags) { const char *start = find_signature(name, internal_doc); const char *end; @@ -597,6 +618,10 @@ _PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_d else end = NULL; if (!end) { + start = signature_from_flags(flags); + if (start) { + return PyUnicode_FromString(start); + } Py_RETURN_NONE; } @@ -1429,7 +1454,7 @@ type_get_doc(PyTypeObject *type, void *context) static PyObject * type_get_text_signature(PyTypeObject *type, void *context) { - return _PyType_GetTextSignatureFromInternalDoc(type->tp_name, type->tp_doc); + return _PyType_GetTextSignatureFromInternalDoc(type->tp_name, type->tp_doc, 0); } static int diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 66815c72ffbc63..c64d391bae13bd 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -419,6 +419,7 @@ Modules/_testbuffer.c staticarray_init kwlist - Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - +Modules/_testcapi/docstring.c - DocStringNoSignatureTest - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - From 637f7ff2c60f262659da0334f1cb672bd361f398 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 11 Aug 2023 18:03:53 +0200 Subject: [PATCH 108/598] Docs: Document PyBUF_MAX_NDIM (#107865) --- Doc/c-api/buffer.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8ca1c190dab9a9..ba391a5279f205 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -161,10 +161,14 @@ a buffer, see :c:func:`PyObject_GetBuffer`. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. + The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. + .. :c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. .. c:member:: Py_ssize_t *shape From 52e0797f8e1c631eecf24cb3f997ace336f52271 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 11 Aug 2023 17:19:19 +0100 Subject: [PATCH 109/598] Extend _sqrtprod() to cover the full range of inputs. Add tests. (GH-107855) --- Lib/statistics.py | 24 ++++++++--- Lib/test/test_statistics.py | 79 +++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 93c44f1f13fab7..a8036e9928c464 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -137,6 +137,7 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod +from math import isfinite, isinf from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict @@ -1005,14 +1006,25 @@ def _mean_stdev(data): return float(xbar), float(xbar) / float(ss) def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with high accuracy." - # Square root differential correction: - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." h = sqrt(x * y) + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h if not h: - return 0.0 - x = sumprod((x, h), (y, -h)) - return h + x / (2.0 * h) + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) # === Statistics for relations between two inputs === diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index f0fa6454b1f91a..aa2cf2b1edc584 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -28,6 +28,12 @@ # === Helper functions and class === +# Test copied from Lib/test/test_math.py +# detect evidence of double-rounding: fsum is not always correctly +# rounded on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -2564,6 +2570,79 @@ def test_different_scales(self): self.assertAlmostEqual(statistics.correlation(x, y), 1) self.assertAlmostEqual(statistics.covariance(x, y), 0.1) + def test_sqrtprod_helper_function_fundamentals(self): + # Verify that results are close to sqrt(x * y) + for i in range(100): + x = random.expovariate() + y = random.expovariate() + expected = math.sqrt(x * y) + actual = statistics._sqrtprod(x, y) + with self.subTest(x=x, y=y, expected=expected, actual=actual): + self.assertAlmostEqual(expected, actual) + + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + # Test that range extremes avoid underflow and overflow + smallest = sys.float_info.min * sys.float_info.epsilon + self.assertEqual(statistics._sqrtprod(smallest, smallest), smallest) + biggest = sys.float_info.max + self.assertEqual(statistics._sqrtprod(biggest, biggest), biggest) + + # Check special values and the sign of the result + special_values = [0.0, -0.0, 1.0, -1.0, 4.0, -4.0, + math.nan, -math.nan, math.inf, -math.inf] + for x, y in itertools.product(special_values, repeat=2): + try: + expected = math.sqrt(x * y) + except ValueError: + expected = 'ValueError' + try: + actual = statistics._sqrtprod(x, y) + except ValueError: + actual = 'ValueError' + with self.subTest(x=x, y=y, expected=expected, actual=actual): + if isinstance(expected, str) and expected == 'ValueError': + self.assertEqual(actual, 'ValueError') + continue + self.assertIsInstance(actual, float) + if math.isnan(expected): + self.assertTrue(math.isnan(actual)) + continue + self.assertEqual(actual, expected) + self.assertEqual(sign(actual), sign(expected)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Allow for a weaker sumprod() implmentation + def test_sqrtprod_helper_function_improved_accuracy(self): + # Test a known example where accuracy is improved + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + def reference_value(x: float, y: float) -> float: + x = decimal.Decimal(x) + y = decimal.Decimal(y) + with decimal.localcontext() as ctx: + ctx.prec = 200 + return float((x * y).sqrt()) + + # Verify that the new function with improved accuracy + # agrees with a reference value more often than old version. + new_agreements = 0 + old_agreements = 0 + for i in range(10_000): + x = random.expovariate() + y = random.expovariate() + new = statistics._sqrtprod(x, y) + old = math.sqrt(x * y) + ref = reference_value(x, y) + new_agreements += (new == ref) + old_agreements += (old == ref) + self.assertGreater(new_agreements, old_agreements) def test_correlation_spearman(self): # https://statistics.laerd.com/statistical-guides/spearmans-rank-order-correlation-statistical-guide-2.php From caa41a4f1db0112690cf610bab7d9c6dce9ff1ce Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 11 Aug 2023 17:42:01 +0100 Subject: [PATCH 110/598] gh-105481: split opcode_ids.h out of opcode.h so that it can be generated separately (#107866) --- .gitattributes | 1 + Include/opcode.h | 221 +---------------------------- Include/opcode_ids.h | 235 +++++++++++++++++++++++++++++++ Makefile.pre.in | 2 + PCbuild/regen.targets | 2 +- Tools/build/generate_opcode_h.py | 61 ++++++-- 6 files changed, 286 insertions(+), 236 deletions(-) create mode 100644 Include/opcode_ids.h diff --git a/.gitattributes b/.gitattributes index 1f2f8a4cd8735a..4a072bc990f0fb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -75,6 +75,7 @@ Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/opcode.h generated +Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated diff --git a/Include/opcode.h b/Include/opcode.h index b3d6cba63096c8..e5c42d5a718286 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -6,227 +6,8 @@ extern "C" { #endif +#include "opcode_ids.h" -/* Instruction opcodes for compiled code */ -#define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define INTERPRETER_EXIT 3 -#define END_FOR 4 -#define END_SEND 5 -#define TO_BOOL 6 -#define NOP 9 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define EXIT_INIT_CHECK 16 -#define RESERVED 17 -#define MAKE_FUNCTION 24 -#define BINARY_SUBSCR 25 -#define BINARY_SLICE 26 -#define STORE_SLICE 27 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define CHECK_EXC_MATCH 36 -#define CHECK_EG_MATCH 37 -#define FORMAT_SIMPLE 40 -#define FORMAT_WITH_SPEC 41 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define CLEANUP_THROW 55 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define RETURN_VALUE 83 -#define SETUP_ANNOTATIONS 85 -#define LOAD_LOCALS 87 -#define POP_EXCEPT 89 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define RETURN_CONST 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define LOAD_FAST_CHECK 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define BUILD_SLICE 133 -#define JUMP_BACKWARD_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define JUMP_BACKWARD 140 -#define LOAD_SUPER_ATTR 141 -#define CALL_FUNCTION_EX 142 -#define LOAD_FAST_AND_CLEAR 143 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define COPY_FREE_VARS 149 -#define YIELD_VALUE 150 -#define RESUME 151 -#define MATCH_CLASS 152 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define CONVERT_VALUE 158 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define LOAD_FAST_LOAD_FAST 168 -#define STORE_FAST_LOAD_FAST 169 -#define STORE_FAST_STORE_FAST 170 -#define CALL 171 -#define KW_NAMES 172 -#define CALL_INTRINSIC_1 173 -#define CALL_INTRINSIC_2 174 -#define LOAD_FROM_DICT_OR_GLOBALS 175 -#define LOAD_FROM_DICT_OR_DEREF 176 -#define SET_FUNCTION_ATTRIBUTE 177 -#define ENTER_EXECUTOR 230 -#define MIN_INSTRUMENTED_OPCODE 237 -#define INSTRUMENTED_LOAD_SUPER_ATTR 237 -#define INSTRUMENTED_POP_JUMP_IF_NONE 238 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 -#define INSTRUMENTED_RESUME 240 -#define INSTRUMENTED_CALL 241 -#define INSTRUMENTED_RETURN_VALUE 242 -#define INSTRUMENTED_YIELD_VALUE 243 -#define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_JUMP_FORWARD 245 -#define INSTRUMENTED_JUMP_BACKWARD 246 -#define INSTRUMENTED_RETURN_CONST 247 -#define INSTRUMENTED_FOR_ITER 248 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_END_FOR 251 -#define INSTRUMENTED_END_SEND 252 -#define INSTRUMENTED_INSTRUCTION 253 -#define INSTRUMENTED_LINE 254 -#define SETUP_FINALLY 256 -#define SETUP_CLEANUP 257 -#define SETUP_WITH 258 -#define POP_BLOCK 259 -#define JUMP 260 -#define JUMP_NO_INTERRUPT 261 -#define LOAD_METHOD 262 -#define LOAD_SUPER_METHOD 263 -#define LOAD_ZERO_SUPER_METHOD 264 -#define LOAD_ZERO_SUPER_ATTR 265 -#define STORE_FAST_MAYBE_NULL 266 -#define LOAD_CLOSURE 267 -#define TO_BOOL_ALWAYS_TRUE 7 -#define TO_BOOL_BOOL 8 -#define TO_BOOL_INT 10 -#define TO_BOOL_LIST 13 -#define TO_BOOL_NONE 14 -#define TO_BOOL_STR 18 -#define BINARY_OP_MULTIPLY_INT 19 -#define BINARY_OP_ADD_INT 20 -#define BINARY_OP_SUBTRACT_INT 21 -#define BINARY_OP_MULTIPLY_FLOAT 22 -#define BINARY_OP_ADD_FLOAT 23 -#define BINARY_OP_SUBTRACT_FLOAT 28 -#define BINARY_OP_ADD_UNICODE 29 -#define BINARY_OP_INPLACE_ADD_UNICODE 34 -#define BINARY_SUBSCR_DICT 38 -#define BINARY_SUBSCR_GETITEM 39 -#define BINARY_SUBSCR_LIST_INT 42 -#define BINARY_SUBSCR_STR_INT 43 -#define BINARY_SUBSCR_TUPLE_INT 44 -#define STORE_SUBSCR_DICT 45 -#define STORE_SUBSCR_LIST_INT 46 -#define SEND_GEN 47 -#define UNPACK_SEQUENCE_TWO_TUPLE 48 -#define UNPACK_SEQUENCE_TUPLE 56 -#define UNPACK_SEQUENCE_LIST 57 -#define STORE_ATTR_INSTANCE_VALUE 58 -#define STORE_ATTR_SLOT 59 -#define STORE_ATTR_WITH_HINT 62 -#define LOAD_GLOBAL_MODULE 63 -#define LOAD_GLOBAL_BUILTIN 64 -#define LOAD_SUPER_ATTR_ATTR 65 -#define LOAD_SUPER_ATTR_METHOD 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 70 -#define LOAD_ATTR_WITH_HINT 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_CLASS 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_LAZY_DICT 81 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 -#define COMPARE_OP_FLOAT 86 -#define COMPARE_OP_INT 88 -#define COMPARE_OP_STR 111 -#define FOR_ITER_LIST 112 -#define FOR_ITER_TUPLE 113 -#define FOR_ITER_RANGE 132 -#define FOR_ITER_GEN 136 -#define CALL_BOUND_METHOD_EXACT_ARGS 148 -#define CALL_PY_EXACT_ARGS 153 -#define CALL_PY_WITH_DEFAULTS 154 -#define CALL_NO_KW_TYPE_1 155 -#define CALL_NO_KW_STR_1 159 -#define CALL_NO_KW_TUPLE_1 160 -#define CALL_BUILTIN_CLASS 161 -#define CALL_NO_KW_BUILTIN_O 166 -#define CALL_NO_KW_BUILTIN_FAST 167 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 -#define CALL_NO_KW_LEN 179 -#define CALL_NO_KW_ISINSTANCE 180 -#define CALL_NO_KW_LIST_APPEND 181 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 #define NB_ADD 0 #define NB_AND 1 diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h new file mode 100644 index 00000000000000..2d9d24cca4542f --- /dev/null +++ b/Include/opcode_ids.h @@ -0,0 +1,235 @@ +// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py + +#ifndef Py_OPCODE_IDS_H +#define Py_OPCODE_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* Instruction opcodes for compiled code */ +#define CACHE 0 +#define POP_TOP 1 +#define PUSH_NULL 2 +#define INTERPRETER_EXIT 3 +#define END_FOR 4 +#define END_SEND 5 +#define TO_BOOL 6 +#define NOP 9 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define EXIT_INIT_CHECK 16 +#define RESERVED 17 +#define MAKE_FUNCTION 24 +#define BINARY_SUBSCR 25 +#define BINARY_SLICE 26 +#define STORE_SLICE 27 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define PUSH_EXC_INFO 35 +#define CHECK_EXC_MATCH 36 +#define CHECK_EG_MATCH 37 +#define FORMAT_SIMPLE 40 +#define FORMAT_WITH_SPEC 41 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define BEFORE_WITH 53 +#define END_ASYNC_FOR 54 +#define CLEANUP_THROW 55 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define LOAD_BUILD_CLASS 71 +#define LOAD_ASSERTION_ERROR 74 +#define RETURN_GENERATOR 75 +#define RETURN_VALUE 83 +#define SETUP_ANNOTATIONS 85 +#define LOAD_LOCALS 87 +#define POP_EXCEPT 89 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define SWAP 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define POP_JUMP_IF_FALSE 114 +#define POP_JUMP_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define COPY 120 +#define RETURN_CONST 121 +#define BINARY_OP 122 +#define SEND 123 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define LOAD_FAST_CHECK 127 +#define POP_JUMP_IF_NOT_NONE 128 +#define POP_JUMP_IF_NONE 129 +#define RAISE_VARARGS 130 +#define GET_AWAITABLE 131 +#define BUILD_SLICE 133 +#define JUMP_BACKWARD_NO_INTERRUPT 134 +#define MAKE_CELL 135 +#define LOAD_DEREF 137 +#define STORE_DEREF 138 +#define DELETE_DEREF 139 +#define JUMP_BACKWARD 140 +#define LOAD_SUPER_ATTR 141 +#define CALL_FUNCTION_EX 142 +#define LOAD_FAST_AND_CLEAR 143 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define COPY_FREE_VARS 149 +#define YIELD_VALUE 150 +#define RESUME 151 +#define MATCH_CLASS 152 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define CONVERT_VALUE 158 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#define LOAD_FAST_LOAD_FAST 168 +#define STORE_FAST_LOAD_FAST 169 +#define STORE_FAST_STORE_FAST 170 +#define CALL 171 +#define KW_NAMES 172 +#define CALL_INTRINSIC_1 173 +#define CALL_INTRINSIC_2 174 +#define LOAD_FROM_DICT_OR_GLOBALS 175 +#define LOAD_FROM_DICT_OR_DEREF 176 +#define SET_FUNCTION_ATTRIBUTE 177 +#define ENTER_EXECUTOR 230 +#define MIN_INSTRUMENTED_OPCODE 237 +#define INSTRUMENTED_LOAD_SUPER_ATTR 237 +#define INSTRUMENTED_POP_JUMP_IF_NONE 238 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 +#define INSTRUMENTED_RESUME 240 +#define INSTRUMENTED_CALL 241 +#define INSTRUMENTED_RETURN_VALUE 242 +#define INSTRUMENTED_YIELD_VALUE 243 +#define INSTRUMENTED_CALL_FUNCTION_EX 244 +#define INSTRUMENTED_JUMP_FORWARD 245 +#define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_RETURN_CONST 247 +#define INSTRUMENTED_FOR_ITER 248 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_END_FOR 251 +#define INSTRUMENTED_END_SEND 252 +#define INSTRUMENTED_INSTRUCTION 253 +#define INSTRUMENTED_LINE 254 +#define SETUP_FINALLY 256 +#define SETUP_CLEANUP 257 +#define SETUP_WITH 258 +#define POP_BLOCK 259 +#define JUMP 260 +#define JUMP_NO_INTERRUPT 261 +#define LOAD_METHOD 262 +#define LOAD_SUPER_METHOD 263 +#define LOAD_ZERO_SUPER_METHOD 264 +#define LOAD_ZERO_SUPER_ATTR 265 +#define STORE_FAST_MAYBE_NULL 266 +#define LOAD_CLOSURE 267 +#define TO_BOOL_ALWAYS_TRUE 7 +#define TO_BOOL_BOOL 8 +#define TO_BOOL_INT 10 +#define TO_BOOL_LIST 13 +#define TO_BOOL_NONE 14 +#define TO_BOOL_STR 18 +#define BINARY_OP_MULTIPLY_INT 19 +#define BINARY_OP_ADD_INT 20 +#define BINARY_OP_SUBTRACT_INT 21 +#define BINARY_OP_MULTIPLY_FLOAT 22 +#define BINARY_OP_ADD_FLOAT 23 +#define BINARY_OP_SUBTRACT_FLOAT 28 +#define BINARY_OP_ADD_UNICODE 29 +#define BINARY_OP_INPLACE_ADD_UNICODE 34 +#define BINARY_SUBSCR_DICT 38 +#define BINARY_SUBSCR_GETITEM 39 +#define BINARY_SUBSCR_LIST_INT 42 +#define BINARY_SUBSCR_STR_INT 43 +#define BINARY_SUBSCR_TUPLE_INT 44 +#define STORE_SUBSCR_DICT 45 +#define STORE_SUBSCR_LIST_INT 46 +#define SEND_GEN 47 +#define UNPACK_SEQUENCE_TWO_TUPLE 48 +#define UNPACK_SEQUENCE_TUPLE 56 +#define UNPACK_SEQUENCE_LIST 57 +#define STORE_ATTR_INSTANCE_VALUE 58 +#define STORE_ATTR_SLOT 59 +#define STORE_ATTR_WITH_HINT 62 +#define LOAD_GLOBAL_MODULE 63 +#define LOAD_GLOBAL_BUILTIN 64 +#define LOAD_SUPER_ATTR_ATTR 65 +#define LOAD_SUPER_ATTR_METHOD 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 70 +#define LOAD_ATTR_WITH_HINT 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_CLASS 76 +#define LOAD_ATTR_PROPERTY 77 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_LAZY_DICT 81 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 +#define COMPARE_OP_FLOAT 86 +#define COMPARE_OP_INT 88 +#define COMPARE_OP_STR 111 +#define FOR_ITER_LIST 112 +#define FOR_ITER_TUPLE 113 +#define FOR_ITER_RANGE 132 +#define FOR_ITER_GEN 136 +#define CALL_BOUND_METHOD_EXACT_ARGS 148 +#define CALL_PY_EXACT_ARGS 153 +#define CALL_PY_WITH_DEFAULTS 154 +#define CALL_NO_KW_TYPE_1 155 +#define CALL_NO_KW_STR_1 159 +#define CALL_NO_KW_TUPLE_1 160 +#define CALL_BUILTIN_CLASS 161 +#define CALL_NO_KW_BUILTIN_O 166 +#define CALL_NO_KW_BUILTIN_FAST 167 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 +#define CALL_NO_KW_LEN 179 +#define CALL_NO_KW_ISINSTANCE 180 +#define CALL_NO_KW_LIST_APPEND 181 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_IDS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index d8fdb34747011d..52236f7924503d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1431,9 +1431,11 @@ regen-opcode: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ $(srcdir)/Lib/_opcode_metadata.py \ + $(srcdir)/Include/opcode_ids.h.new \ $(srcdir)/Include/opcode.h.new \ $(srcdir)/Python/opcode_targets.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new + $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 99cfff5acc0baf..c1189c883b667c 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 3a817326c94cbb..67f4a2c2d5d76f 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -6,7 +6,7 @@ SCRIPT_NAME = "Tools/build/generate_opcode_h.py" PYTHON_OPCODE = "Lib/opcode.py" -header = f""" +opcode_h_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} #ifndef Py_OPCODE_H @@ -15,11 +15,11 @@ extern "C" {{ #endif +#include "opcode_ids.h" -/* Instruction opcodes for compiled code */ """.lstrip() -footer = """ +opcode_h_footer = """ #ifdef __cplusplus } @@ -27,6 +27,27 @@ #endif /* !Py_OPCODE_H */ """ +opcode_ids_h_header = f""" +// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} + +#ifndef Py_OPCODE_IDS_H +#define Py_OPCODE_IDS_H +#ifdef __cplusplus +extern "C" {{ +#endif + + +/* Instruction opcodes for compiled code */ +""".lstrip() + +opcode_ids_h_footer = """ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_IDS_H */ +""" + internal_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} @@ -63,9 +84,10 @@ def get_python_module_dict(filename): def main(opcode_py, _opcode_metadata_py='Lib/_opcode_metadata.py', - outfile='Include/opcode.h', + opcode_ids_h='Include/opcode_ids.h', + opcode_h='Include/opcode.h', opcode_targets_h='Python/opcode_targets.h', - internaloutfile='Include/internal/pycore_opcode.h'): + internal_opcode_h='Include/internal/pycore_opcode.h'): _opcode_metadata = get_python_module_dict(_opcode_metadata_py) @@ -91,9 +113,8 @@ def main(opcode_py, opname_including_specialized[next_op] = name used[next_op] = True - with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: - fobj.write(header) - iobj.write(internal_header) + with open(opcode_ids_h, 'w') as fobj: + fobj.write(opcode_ids_h_header) for name in opname: if name in opmap: @@ -107,6 +128,20 @@ def main(opcode_py, for name, op in specialized_opmap.items(): fobj.write(DEFINE.format(name, op)) + fobj.write(opcode_ids_h_footer) + + with open(opcode_h, 'w') as fobj: + fobj.write(opcode_h_header) + + fobj.write("\n") + for i, (op, _) in enumerate(opcode["_nb_ops"]): + fobj.write(DEFINE.format(op, i)) + + fobj.write(opcode_h_footer) + + with open(internal_opcode_h, 'w') as iobj: + iobj.write(internal_header) + iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") @@ -129,10 +164,6 @@ def main(opcode_py, iobj.write("};\n") iobj.write("#endif // NEED_OPCODE_TABLES\n") - fobj.write("\n") - for i, (op, _) in enumerate(opcode["_nb_ops"]): - fobj.write(DEFINE.format(op, i)) - iobj.write("\n") iobj.write(f"\nextern const char *const _PyOpcode_OpName[{NUM_OPCODES}];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") @@ -151,7 +182,6 @@ def main(opcode_py, iobj.write(f" case {i}: \\\n") iobj.write(" ;\n") - fobj.write(footer) iobj.write(internal_footer) with open(opcode_targets_h, "w") as f: @@ -164,8 +194,9 @@ def main(opcode_py, f.write(",\n".join([f" &&{s}" for s in targets])) f.write("\n};\n") - print(f"{outfile} regenerated from {opcode_py}") + print(f"{opcode_h} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], + sys.argv[5], sys.argv[6]) From 5f7d4ecf301ef12eb1d1d347add054f4fcd8fc5c Mon Sep 17 00:00:00 2001 From: Andrew Geng Date: Fri, 11 Aug 2023 13:44:18 -0400 Subject: [PATCH 111/598] gh-106558: break ref cycles through exceptions in multiprocessing manager (#106559) --- Lib/multiprocessing/managers.py | 10 ++++- Lib/test/_test_multiprocessing.py | 38 +++++++++++++++++++ ...-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst | 3 ++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index b6534939b4d98b..273c22a7654f05 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}): kind, result = c.recv() if kind == '#RETURN': return result - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def convert_to_error(kind, result): if kind == '#ERROR': @@ -833,7 +836,10 @@ def _callmethod(self, methodname, args=(), kwds={}): conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def _getvalue(self): ''' diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c1f9487ae80511..f881a5d4674699 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3149,6 +3149,44 @@ def test_rapid_restart(self): if hasattr(manager, "shutdown"): self.addCleanup(manager.shutdown) + +class FakeConnection: + def send(self, payload): + pass + + def recv(self): + return '#ERROR', pyqueue.Empty() + +class TestManagerExceptions(unittest.TestCase): + # Issue 106558: Manager exceptions avoids creating cyclic references. + def setUp(self): + self.mgr = multiprocessing.Manager() + + def tearDown(self): + self.mgr.shutdown() + self.mgr.join() + + def test_queue_get(self): + queue = self.mgr.Queue() + if gc.isenabled(): + gc.disable() + self.addCleanup(gc.enable) + try: + queue.get_nowait() + except pyqueue.Empty as e: + wr = weakref.ref(e) + self.assertEqual(wr(), None) + + def test_dispatch(self): + if gc.isenabled(): + gc.disable() + self.addCleanup(gc.enable) + try: + multiprocessing.managers.dispatch(FakeConnection(), None, None) + except pyqueue.Empty as e: + wr = weakref.ref(e) + self.assertEqual(wr(), None) + # # # diff --git a/Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst b/Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst new file mode 100644 index 00000000000000..8fe677f5d84b5f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst @@ -0,0 +1,3 @@ +Remove ref cycle in callers of +:func:`~multiprocessing.managers.convert_to_error` by deleting ``result`` +from scope in a ``finally`` block. From a39f0a350662f1978104ee1136472d784aa6f29c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 11 Aug 2023 20:51:36 +0300 Subject: [PATCH 112/598] gh-107782: Pydoc: fall back to __text_signature__ if inspect.signature() fails (GH-107786) It allows to show signatures which are not representable in Python, e.g. for getattr and dict.pop. --- Lib/pydoc.py | 78 +++++++++---------- Lib/test/test_pydoc.py | 54 +++++++++++++ ...-08-08-19-57-45.gh-issue-107782.mInjFE.rst | 2 + 3 files changed, 94 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 185f09e603df2e..c9a55799b39f0c 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -197,6 +197,24 @@ def splitdoc(doc): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) +def _getargspec(object): + try: + signature = inspect.signature(object) + if signature: + return str(signature) + except (ValueError, TypeError): + argspec = getattr(object, '__text_signature__', None) + if argspec: + if argspec[:2] == '($': + argspec = '(' + argspec[2:] + if getattr(object, '__self__', None) is not None: + # Strip the bound argument. + m = re.match(r'\(\w+(?:(?=\))|,\s*(?:/(?:(?=\))|,\s*))?)', argspec) + if m: + argspec = '(' + argspec[m.end():] + return argspec + return None + def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ @@ -1003,14 +1021,9 @@ def spilldata(msg, attrs, predicate): title = title + '(%s)' % ', '.join(parents) decl = '' - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if argspec and argspec != '()': - decl = name + self.escape(argspec) + '\n\n' + argspec = _getargspec(object) + if argspec and argspec != '()': + decl = name + self.escape(argspec) + '\n\n' doc = getdoc(object) if decl: @@ -1063,18 +1076,13 @@ def docroutine(self, object, name=None, mod=None, anchor, name, reallink) argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = '%s lambda ' % name - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = '%s lambda ' % name + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' @@ -1321,14 +1329,9 @@ def makename(c, m=object.__module__): contents = [] push = contents.append - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if argspec and argspec != '()': - push(name + argspec + '\n') + argspec = _getargspec(object) + if argspec and argspec != '()': + push(name + argspec + '\n') doc = getdoc(object) if doc: @@ -1492,18 +1495,13 @@ def docroutine(self, object, name=None, mod=None, cl=None): argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = self.bold(name) + ' lambda ' - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = self.bold(name) + ' lambda ' + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' decl = asyncqualifier + title + argspec + note diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 8df8b608cf959d..fe4e37d4858c85 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1230,6 +1230,60 @@ def test_bound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__class_getitem__), "__class_getitem__(object, /) method of builtins.type instance") + def test_module_level_callable_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(getattr), + "getattr(object, name, default=, /)") + + def test_builtin_staticmethod_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(str.maketrans), + "maketrans(x, y=, z=, /)") + + def test_unbound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(dict.pop), + "pop(self, key, default=, /)") + + def test_bound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line({}.pop), + "pop(key, default=, /) " + "method of builtins.dict instance") + + def test_overridden_text_signature(self): + class C: + def meth(*args, **kwargs): + pass + @classmethod + def cmeth(*args, **kwargs): + pass + @staticmethod + def smeth(*args, **kwargs): + pass + for text_signature, unbound, bound in [ + ("($slf)", "(slf, /)", "()"), + ("($slf, /)", "(slf, /)", "()"), + ("($slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("($slf, /, arg=)", "(slf, /, arg=)", "(arg=)"), + ("($slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("($slf, arg=, /)", "(slf, arg=, /)", "(arg=, /)"), + ("(/, slf, arg)", "(/, slf, arg)", "(/, slf, arg)"), + ("(/, slf, arg=)", "(/, slf, arg=)", "(/, slf, arg=)"), + ("(slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("(slf, /, arg=)", "(slf, /, arg=)", "(arg=)"), + ("(slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("(slf, arg=, /)", "(slf, arg=, /)", "(arg=, /)"), + ]: + with self.subTest(text_signature): + C.meth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.meth), + "meth" + unbound) + self.assertEqual(self._get_summary_line(C().meth), + "meth" + bound + " method of test.test_pydoc.C instance") + C.cmeth.__func__.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.cmeth), + "cmeth" + bound + " method of builtins.type instance") + C.smeth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.smeth), + "smeth" + unbound) + @requires_docstrings def test_staticmethod(self): class X: diff --git a/Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst b/Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst new file mode 100644 index 00000000000000..fb8a50de3a9eee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst @@ -0,0 +1,2 @@ +:mod:`pydoc` is now able to show signatures which are not representable in +Python, e.g. for ``getattr`` and ``dict.pop``. From 04cc01453db2f0af72a06440831637f8bf512daf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 11 Aug 2023 21:13:46 +0300 Subject: [PATCH 113/598] gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) * Strings with length from 2**31-1 to 2**32-2 always caused MemoryError, it doesn't matter how much memory is available. * Strings with length exactly 2**32-1 caused OSError. * Strings longer than 2**32-1 characters were truncated due to integer overflow bug. * Strings containing the null character were truncated at the first null character. Now strings longer than 2**31-1 characters caused OverflowError and the null character is allowed. --- Lib/test/test_ntpath.py | 1 + ...-07-18-13-01-26.gh-issue-106844.mci4xO.rst | 1 + Modules/_winapi.c | 34 ++++++++++++++----- Modules/clinic/_winapi.c.h | 12 +++---- 4 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 538d758624c9d6..78e1cb582512b0 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1036,6 +1036,7 @@ def test_path_normcase(self): self._check_function(self.path.normcase) if sys.platform == 'win32': self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') + self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst new file mode 100644 index 00000000000000..1fdf162ef4ecdd --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst @@ -0,0 +1 @@ +Fix integer overflow and truncating by the null character in :func:`!_winapi.LCMapStringEx` which affects :func:`ntpath.normcase`. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 0edb36d18dd3ff..eec33499b983fe 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1539,40 +1539,56 @@ _winapi.LCMapStringEx locale: LPCWSTR flags: DWORD - src: LPCWSTR + src: unicode [clinic start generated code]*/ static PyObject * _winapi_LCMapStringEx_impl(PyObject *module, LPCWSTR locale, DWORD flags, - LPCWSTR src) -/*[clinic end generated code: output=cf4713d80e2b47c9 input=9fe26f95d5ab0001]*/ + PyObject *src) +/*[clinic end generated code: output=b90e6b26e028ff0a input=3e3dcd9b8164012f]*/ { if (flags & (LCMAP_SORTHANDLE | LCMAP_HASH | LCMAP_BYTEREV | LCMAP_SORTKEY)) { return PyErr_Format(PyExc_ValueError, "unsupported flags"); } - int dest_size = LCMapStringEx(locale, flags, src, -1, NULL, 0, + Py_ssize_t src_size; + wchar_t *src_ = PyUnicode_AsWideCharString(src, &src_size); + if (!src_) { + return NULL; + } + if (src_size > INT_MAX) { + PyMem_Free(src_); + PyErr_SetString(PyExc_OverflowError, "input string is too long"); + return NULL; + } + + int dest_size = LCMapStringEx(locale, flags, src_, (int)src_size, NULL, 0, NULL, NULL, 0); - if (dest_size == 0) { - return PyErr_SetFromWindowsErr(0); + if (dest_size <= 0) { + DWORD error = GetLastError(); + PyMem_Free(src_); + return PyErr_SetFromWindowsErr(error); } wchar_t* dest = PyMem_NEW(wchar_t, dest_size); if (dest == NULL) { + PyMem_Free(src_); return PyErr_NoMemory(); } - int nmapped = LCMapStringEx(locale, flags, src, -1, dest, dest_size, + int nmapped = LCMapStringEx(locale, flags, src_, (int)src_size, dest, dest_size, NULL, NULL, 0); - if (nmapped == 0) { + if (nmapped <= 0) { DWORD error = GetLastError(); + PyMem_Free(src_); PyMem_DEL(dest); return PyErr_SetFromWindowsErr(error); } - PyObject *ret = PyUnicode_FromWideChar(dest, dest_size - 1); + PyMem_Free(src_); + PyObject *ret = PyUnicode_FromWideChar(dest, nmapped); PyMem_DEL(dest); return ret; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 8f46b8f1095e98..35ac053547121c 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -884,7 +884,7 @@ PyDoc_STRVAR(_winapi_LCMapStringEx__doc__, static PyObject * _winapi_LCMapStringEx_impl(PyObject *module, LPCWSTR locale, DWORD flags, - LPCWSTR src); + PyObject *src); static PyObject * _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -911,16 +911,16 @@ _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static const char * const _keywords[] = {"locale", "flags", "src", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "O&kO&:LCMapStringEx", + .format = "O&kU:LCMapStringEx", .kwtuple = KWTUPLE, }; #undef KWTUPLE LPCWSTR locale = NULL; DWORD flags; - LPCWSTR src = NULL; + PyObject *src; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - _PyUnicode_WideCharString_Converter, &locale, &flags, _PyUnicode_WideCharString_Converter, &src)) { + _PyUnicode_WideCharString_Converter, &locale, &flags, &src)) { goto exit; } return_value = _winapi_LCMapStringEx_impl(module, locale, flags, src); @@ -928,8 +928,6 @@ _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: /* Cleanup for locale */ PyMem_Free((void *)locale); - /* Cleanup for src */ - PyMem_Free((void *)src); return return_value; } @@ -1480,4 +1478,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=f32fe6ecdbffd74d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ff91ab5cae8961dd input=a9049054013a1b77]*/ From 66e4edd7346b1cd65ddff6da890a0d725e325116 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 11 Aug 2023 12:42:26 -0600 Subject: [PATCH 114/598] gh-91051: fix segfault when using all 8 type watchers (#107853) --- Doc/c-api/typeobj.rst | 4 ++-- Doc/includes/typestruct.h | 2 +- Include/cpython/object.h | 2 +- Lib/test/test_capi/test_watchers.py | 12 ++++++++++++ .../2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst | 2 ++ 5 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 221a05b1922404..faa183e27fcfa2 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | unsigned char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -2141,7 +2141,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) -.. c:member:: char PyTypeObject.tp_watched +.. c:member:: unsigned char PyTypeObject.tp_watched Internal. Do not use. diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index f0ad1e47cb0d86..ec939c28831c33 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -82,5 +82,5 @@ typedef struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; } PyTypeObject; diff --git a/Include/cpython/object.h b/Include/cpython/object.h index fd45fa57e6282b..5f8b1f7c195501 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -227,7 +227,7 @@ struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; }; /* This struct is used by the specializer diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 93f6ef752d0663..10b76e163bfb21 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -294,6 +294,18 @@ class C2: pass C2.hmm = "baz" self.assert_events([C1, [C2]]) + def test_all_watchers(self): + class C: pass + with ExitStack() as stack: + last_wid = -1 + # don't make assumptions about how many watchers are already + # registered, just go until we reach the max ID + while last_wid < self.TYPE_MAX_WATCHERS - 1: + last_wid = stack.enter_context(self.watcher()) + self.watch(last_wid, C) + C.foo = "bar" + self.assert_events([C]) + def test_watch_non_type(self): with self.watcher() as wid: with self.assertRaisesRegex(ValueError, r"Cannot watch non-type"): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst new file mode 100644 index 00000000000000..b4b90ad4ea0ecc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst @@ -0,0 +1,2 @@ +Fix abort / segfault when using all eight type watcher slots, on platforms +where ``char`` is signed by default. From 666b68e8f252e3c6238d6eed1fc82937a774316f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 11 Aug 2023 20:05:56 +0100 Subject: [PATCH 115/598] GH-106485: Create object's dict-values instead of creating __dict__, when we can. (GH-107843) --- Objects/dictobject.c | 4 +--- Objects/object.c | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 36375f50646fd4..f9701f6b4b09ad 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5762,10 +5762,8 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, assert(dictptr != NULL); dict = *dictptr; if (dict == NULL) { + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); dictkeys_incref(cached); - if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - OBJECT_STAT_INC(dict_materialized_on_request); - } dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; diff --git a/Objects/object.c b/Objects/object.c index d1154eb344fab1..868623a9f7bffc 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1577,6 +1577,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto error_check; } dictptr = &dorv_ptr->dict; + if (*dictptr == NULL) { + if (_PyObject_InitInlineValues(obj, tp) < 0) { + goto done; + } + res = _PyObject_StoreInstanceAttribute( + obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); + goto error_check; + } } else { dictptr = _PyObject_ComputedDictPointer(obj); From d93b4ac2ff7bce07fb1c8805f43838818598191c Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 11 Aug 2023 22:12:11 +0300 Subject: [PATCH 116/598] gh-101162: Forbid using issubclass() with GenericAlias as the 1st arg (GH-103369) --- Lib/test/test_typing.py | 16 ++++++++++++++++ ...023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst | 2 ++ Objects/abstract.c | 2 +- Objects/genericaliasobject.c | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0450a87577ecea..fa39c796197959 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4091,6 +4091,22 @@ class C(Generic[T]): pass with self.assertRaises(TypeError): C[()] + def test_generic_subclass_checks(self): + for typ in [list[int], List[int], + tuple[int, str], Tuple[int, str], + typing.Callable[..., None], + collections.abc.Callable[..., None]]: + with self.subTest(typ=typ): + self.assertRaises(TypeError, issubclass, typ, object) + self.assertRaises(TypeError, issubclass, typ, type) + self.assertRaises(TypeError, issubclass, typ, typ) + self.assertRaises(TypeError, issubclass, object, typ) + + # isinstance is fine: + self.assertTrue(isinstance(typ, object)) + # but, not when the right arg is also a generic: + self.assertRaises(TypeError, isinstance, typ, typ) + def test_init(self): T = TypeVar('T') S = TypeVar('S') diff --git a/Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst b/Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst new file mode 100644 index 00000000000000..e9fadc8f436d9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst @@ -0,0 +1,2 @@ +Forbid using :func:`builtins.issubclass` with :class:`types.GenericAlias` as +the first argument. diff --git a/Objects/abstract.c b/Objects/abstract.c index b4edcec6007710..c113364a88a26a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2812,7 +2812,7 @@ object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls) return -1; } - /* Probably never reached anymore. */ + /* Can be reached when infinite recursion happens. */ return recursive_issubclass(derived, cls); } diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index df8873454aeb36..faf517b66b9350 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -626,6 +626,7 @@ ga_vectorcall(PyObject *self, PyObject *const *args, static const char* const attr_exceptions[] = { "__class__", + "__bases__", "__origin__", "__args__", "__unpacked__", From 2e27da18952c6561f48dab706b5911135cedd7cf Mon Sep 17 00:00:00 2001 From: nahyeon <55136494+nahyeon-an@users.noreply.github.com> Date: Sat, 12 Aug 2023 09:57:35 +0900 Subject: [PATCH 117/598] gh-104469 : Convert _testcapi/vectorcall_limited.c to use AC (gh-107857) --- .../_testcapi/clinic/vectorcall_limited.c.h | 42 +++++++++++++++++++ Modules/_testcapi/vectorcall_limited.c | 30 +++++++++++-- 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 Modules/_testcapi/clinic/vectorcall_limited.c.h diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h new file mode 100644 index 00000000000000..bc89f58b65739b --- /dev/null +++ b/Modules/_testcapi/clinic/vectorcall_limited.c.h @@ -0,0 +1,42 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +#if defined(LIMITED_API_AVAILABLE) + +PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, +"call_vectorcall($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ + {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, + +#endif /* defined(LIMITED_API_AVAILABLE) */ + +#if defined(LIMITED_API_AVAILABLE) + +PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, +"call_vectorcall_method($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ + {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, + +#endif /* defined(LIMITED_API_AVAILABLE) */ + +#ifndef _TESTCAPI_CALL_VECTORCALL_METHODDEF + #define _TESTCAPI_CALL_VECTORCALL_METHODDEF +#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHODDEF) */ + +#ifndef _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + #define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF +#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF) */ +/*[clinic end generated code: output=409028b637aba77b input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index a96925e840121a..9f02c7afa7c971 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,5 +1,6 @@ #define Py_LIMITED_API 0x030c0000 // 3.12 #include "parts.h" +#include "clinic/vectorcall_limited.c.h" #ifdef LIMITED_API_AVAILABLE @@ -7,6 +8,11 @@ /* Test Vectorcall in the limited API */ +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ + static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { return PyUnicode_FromString("tp_call called"); @@ -32,8 +38,16 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) return self; } +/*[clinic input] +_testcapi.call_vectorcall + + callable: object + / +[clinic start generated code]*/ + static PyObject * -call_vectorcall(PyObject* self, PyObject *callable) +_testcapi_call_vectorcall(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -77,8 +91,16 @@ call_vectorcall(PyObject* self, PyObject *callable) return result; } +/*[clinic input] +_testcapi.call_vectorcall_method + + callable: object + / +[clinic start generated code]*/ + static PyObject * -call_vectorcall_method(PyObject* self, PyObject *callable) +_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -153,8 +175,8 @@ static PyType_Spec LimitedVectorCallClass_spec = { }; static PyMethodDef TestMethods[] = { - {"call_vectorcall", call_vectorcall, METH_O}, - {"call_vectorcall_method", call_vectorcall_method, METH_O}, + _TESTCAPI_CALL_VECTORCALL_METHODDEF + _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF {NULL}, }; From 2e1f688fe0f0a612e54c09f5a7027a834dd8b8d5 Mon Sep 17 00:00:00 2001 From: wookie184 Date: Sat, 12 Aug 2023 13:05:22 +0100 Subject: [PATCH 118/598] gh-107891: Fix typo in 3.12 whatsnew (#107892) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8ed435476c9cec..65ca4c332ce35f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1576,7 +1576,7 @@ Changes in the Python API 1,13-1,17: FSTRING_MIDDLE ' end' 1,17-1,18: FSTRING_END '"' - Additionally, there may be some minor behavioral changes as a consecuence of the + Additionally, there may be some minor behavioral changes as a consequence of the changes required to support :pep:`701`. Some of these changes include: * The ``type`` attribute of the tokens emitted when tokenizing some invalid Python From bf707749e87f47b2fee2a208a654511ac318d4b9 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 13 Aug 2023 04:36:46 +0900 Subject: [PATCH 119/598] gh-106797: Remove warning logs from Python/generated_cases.c.h and executor_cases.c.h (gh-107889) gh-106797: Remove warning logs from Python/generated_cases.c.h --- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 6 +++--- Tools/cases_generator/stacking.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a7b5054417ed8f..5e3c84b2b4b612 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1513,7 +1513,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ccf43c727b9e0f..2661a39e047c4d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2251,7 +2251,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; next_instr += 1; DISPATCH(); } @@ -3574,7 +3574,7 @@ assert(descr != NULL); Py_DECREF(owner); attr = Py_NewRef(descr); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; next_instr += 9; DISPATCH(); } @@ -3594,7 +3594,7 @@ assert(descr != NULL); Py_DECREF(owner); attr = Py_NewRef(descr); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; next_instr += 9; DISPATCH(); } diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index d457ce01a8f438..9bb7f468442245 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -61,14 +61,14 @@ def as_terms(self) -> list[tuple[str, str]]: for eff in self.deep: if eff.size: terms.append(("-", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond != "1": + elif eff.cond and eff.cond not in ("0", "1"): terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) elif eff.cond != "0": num -= 1 for eff in self.high: if eff.size: terms.append(("+", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond != "1": + elif eff.cond and eff.cond not in ("0", "1"): terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) elif eff.cond != "0": num += 1 From ee40b3e20d9b8d62a9b36b777dff42db1e9049d5 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 13 Aug 2023 01:46:00 +0200 Subject: [PATCH 120/598] gh-107883: Argument Clinic: Handle full module/class path in Function.fulldisplayname (#107884) Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 54 +++++++++++++++++++++++++++++++++++++++++ Tools/clinic/clinic.py | 13 +++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a649b5fe2201c8..55251b516d2b2f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1513,6 +1513,60 @@ def test_parameters_required_after_star(self): with self.subTest(block=block): self.expect_failure(block, err) + def test_fulldisplayname_class(self): + dataset = ( + ("T", """ + class T "void *" "" + T.__init__ + """), + ("m.T", """ + module m + class m.T "void *" "" + @classmethod + m.T.__new__ + """), + ("m.T.C", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.__init__ + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + + def test_fulldisplayname_meth(self): + dataset = ( + ("func", "func"), + ("m.func", """ + module m + m.func + """), + ("T.meth", """ + class T "void *" "" + T.meth + """), + ("m.T.meth", """ + module m + class m.T "void *" "" + m.T.meth + """), + ("m.T.C.meth", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.meth + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + def test_depr_star_invalid_format_1(self): block = """ module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 70b066cce82fae..2d23f9d12875ff 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2682,9 +2682,16 @@ def displayname(self) -> str: @functools.cached_property def fulldisplayname(self) -> str: - if isinstance(self.module, Module): - return f"{self.module.name}.{self.displayname}" - return self.displayname + parent: Class | Module | Clinic | None + if self.kind.new_or_init: + parent = getattr(self.cls, "parent", None) + else: + parent = self.parent + name = self.displayname + while isinstance(parent, (Module, Class)): + name = f"{parent.name}.{name}" + parent = parent.parent + return name @property def render_parameters(self) -> list[Parameter]: From 7ddc1eaff16fa946516283be82d8b2e2ef315b03 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 13 Aug 2023 12:24:59 +0300 Subject: [PATCH 121/598] Improve `_typing.__doc__` (#107908) --- Modules/_typingmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 59d3a80a9305db..9ea72bf89ce0b2 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -40,7 +40,7 @@ static PyMethodDef typing_methods[] = { }; PyDoc_STRVAR(typing_doc, -"Accelerators for the typing module.\n"); +"Primitives and accelerators for the typing module.\n"); static int _typing_exec(PyObject *m) From 9b75ada6e4232ff03b716b1c5930d1a3ba3b16b8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 13 Aug 2023 12:13:11 +0200 Subject: [PATCH 122/598] gh-107880: Teach Argument Clinic to clone __init__ and __new__ methods (#107885) --- Lib/test/test_clinic.py | 22 +++ ...-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst | 2 + Modules/_testclinic.c | 34 ++++ Modules/clinic/_testclinic_depr_star.c.h | 168 +++++++++++++++++- Tools/clinic/clinic.py | 26 ++- 5 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 55251b516d2b2f..f067a26d1fb3ae 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2973,6 +2973,17 @@ def test_depr_star_new(self): ac_tester.DeprStarNew(None) self.assertEqual(cm.filename, __file__) + def test_depr_star_new_cloned(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarNew.cloned() " + "is deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + obj = ac_tester.DeprStarNew(a=None) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + obj.cloned(None) + self.assertEqual(cm.filename, __file__) + def test_depr_star_init(self): regex = re.escape( "Passing positional arguments to _testclinic.DeprStarInit() is " @@ -2983,6 +2994,17 @@ def test_depr_star_init(self): ac_tester.DeprStarInit(None) self.assertEqual(cm.filename, __file__) + def test_depr_star_init_cloned(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarInit.cloned() " + "is deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + obj = ac_tester.DeprStarInit(a=None) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + obj.cloned(None) + self.assertEqual(cm.filename, __file__) + def test_depr_star_pos0_len1(self): fn = ac_tester.depr_star_pos0_len1 fn(a=None) diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst new file mode 100644 index 00000000000000..fd9d6717f3a33f --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst @@ -0,0 +1,2 @@ +Argument Clinic can now clone :meth:`!__init__` and :meth:`!__new__` +methods. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 8fa3cc83d871b1..c33536234af0bc 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1230,12 +1230,29 @@ depr_star_new_impl(PyTypeObject *type, PyObject *a) return type->tp_alloc(type, 0); } +/*[clinic input] +_testclinic.DeprStarNew.cloned as depr_star_new_clone = _testclinic.DeprStarNew.__new__ +[clinic start generated code]*/ + +static PyObject * +depr_star_new_clone_impl(PyObject *type, PyObject *a) +/*[clinic end generated code: output=3b17bf885fa736bc input=ea659285d5dbec6c]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef depr_star_new_methods[] = { + DEPR_STAR_NEW_CLONE_METHODDEF + {NULL, NULL} +}; + static PyTypeObject DeprStarNew = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_testclinic.DeprStarNew", .tp_basicsize = sizeof(PyObject), .tp_new = depr_star_new, .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = depr_star_new_methods, }; @@ -1254,6 +1271,22 @@ depr_star_init_impl(PyObject *self, PyObject *a) return 0; } +/*[clinic input] +_testclinic.DeprStarInit.cloned as depr_star_init_clone = _testclinic.DeprStarInit.__init__ +[clinic start generated code]*/ + +static PyObject * +depr_star_init_clone_impl(PyObject *self, PyObject *a) +/*[clinic end generated code: output=ddfe8a1b5531e7cc input=561e103fe7f8e94f]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef depr_star_init_methods[] = { + DEPR_STAR_INIT_CLONE_METHODDEF + {NULL, NULL} +}; + static PyTypeObject DeprStarInit = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_testclinic.DeprStarInit", @@ -1261,6 +1294,7 @@ static PyTypeObject DeprStarInit = { .tp_new = PyType_GenericNew, .tp_init = depr_star_init, .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = depr_star_init_methods, }; diff --git a/Modules/clinic/_testclinic_depr_star.c.h b/Modules/clinic/_testclinic_depr_star.c.h index 0c2fa088268c6d..1aa42dd4059777 100644 --- a/Modules/clinic/_testclinic_depr_star.c.h +++ b/Modules/clinic/_testclinic_depr_star.c.h @@ -92,6 +92,89 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(depr_star_new_clone__doc__, +"cloned($self, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" +"is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_NEW_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_new_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_new_clone__doc__}, + +static PyObject * +depr_star_new_clone_impl(PyObject *type, PyObject *a); + +static PyObject * +depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.cloned' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.cloned' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.cloned' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew.cloned()" + " is deprecated. Parameter 'a' will become a keyword-only " + "parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_new_clone_impl(type, a); + +exit: + return return_value; +} + PyDoc_STRVAR(depr_star_init__doc__, "DeprStarInit(a)\n" "--\n" @@ -176,6 +259,89 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(depr_star_init_clone__doc__, +"cloned($self, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to\n" +"_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" +"become a keyword-only parameter in Python 3.14.\n" +""); + +#define DEPR_STAR_INIT_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_init_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_init_clone__doc__}, + +static PyObject * +depr_star_init_clone_impl(PyObject *self, PyObject *a); + +static PyObject * +depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.cloned' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.cloned' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.cloned' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " + "will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_init_clone_impl(self, a); + +exit: + return return_value; +} + PyDoc_STRVAR(depr_star_pos0_len1__doc__, "depr_star_pos0_len1($module, /, a)\n" "--\n" @@ -971,4 +1137,4 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t exit: return return_value; } -/*[clinic end generated code: output=18ab056f6cc06d7e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a16fee4d6742d54 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2d23f9d12875ff..1e0303c77087eb 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4888,13 +4888,25 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) - if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): - fail("'kind' of function and cloned function don't match! " - "(@classmethod/@staticmethod/@coexist)") - function = existing_function.copy( - name=function_name, full_name=full_name, module=module, - cls=cls, c_basename=c_basename, docstring='' - ) + overrides: dict[str, Any] = { + "name": function_name, + "full_name": full_name, + "module": module, + "cls": cls, + "c_basename": c_basename, + "docstring": "", + } + if not (existing_function.kind is self.kind and + existing_function.coexist == self.coexist): + # Allow __new__ or __init__ methods. + if existing_function.kind.new_or_init: + overrides["kind"] = self.kind + # Future enhancement: allow custom return converters + overrides["return_converter"] = CReturnConverter() + else: + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") + function = existing_function.copy(**overrides) self.function = function self.block.signatures.append(function) (cls or module).functions.append(function) From 2b6dc2accc315ce279d259ed39e058a225068531 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Aug 2023 08:01:23 -0500 Subject: [PATCH 123/598] Add another example to the statistics docs (GH-107904) --- Doc/library/kde_example.png | Bin 0 -> 332212 bytes Doc/library/statistics.rst | 57 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 Doc/library/kde_example.png diff --git a/Doc/library/kde_example.png b/Doc/library/kde_example.png new file mode 100644 index 0000000000000000000000000000000000000000..f45048956999741a6889bdfe85c40e1f25e8863b GIT binary patch literal 332212 zcmeEuXIN8fw=GqI2t?@}ETB>YLO?pGsDRiIgix~yBArMlgc3xh=>{Q4w}8?Sq<17y z2}M8xgdRf+y|=(!?B|?wukP>1`S!2-<9mEe0;H_0cfNCuG3J=->CGDk>_-KU($Uee zU%PtgHXR)sn2wG`9drcvO@e5tHE>6dyltRISJ)vu1N?w-Fu#T{Hl{ldJO~I>pAr|MX}7uprUV>Cj!fqa1-(SEv6};s5Irg$Lp z-p_vtvm}cetG%huj8z8ySXmh3CY73KZNJ@`eiQdh`(V4XwalsKnQQq#cE4AfY)&sk zPN=KAV%UfDLXm%c$D~A#=2KNnkS^&zO`G}qQr3uhE`#P5yYRGP2eZiW^r00PHF z+waa-RA?K0gwHShZG{_`4l!>%+&<>-7%(&o?6=&?7bWCgTjIAflLKWk-t33RP5fyn zcP>_S;)#8aF;wJPfW=cE2Kk0PFV9vUGj0>j^}EOQqbyxNnR@`rx^^ZMChj@RsNT z#=(R<0fW%Zk-#Fq1KMWGa&(G=%0$IzusY$o1`oq$^d4C(>XS3k?TCkB8spHjot$krJz8>8Jbd7^&QO~VxqMHPpB`a#=GcA{ix`H-%jjN#1y6d4 zy02SF3y1nKgnr>WcY3$-v5PSuTX`)n%;uonADt)+S)K`H5d}e|3aslc)WD-GlaLf2?9-jXnuVj#^9zcJ_>br{$B;^m_kS9Mo_Q*53JM5F|aw7B!w$v=!cPJ7=E>n+_! zeQ9RRYRAKH9tcMFpuy3<0xtgQ*#7P!XV&M2J&L^0#pZ`3$?2qF@;XV{QGx?0vGGDGDJ~p-)}2(o7V^VSK7&S z{2clNf^cxWV$t@K+ix=2@!91l-iYK?vgnEP^^5hpdIuNBFMv|?85DSE>kR?f!+IW&&r+Sa0#?k$InIIY_ShwL?+wNjR zgB`ASOt zYzCrld`D3g=5+zyB|r}6sadF|J}(31-Qu0DkX?1p95PODc|+VcEzCgwbw2;!?Wm&! ze5dx*prl_TwULRb#_*SPn<;#K3 z!=?PJ3sOcFu5lT46m(AIGOK0-o5gq;(6o37XRnT9(~%u&t;={ZF+@wA^`O~dY-}(9 z3*g=4?^9ma+8ZY9D892rI!c$9qz4uO3&kc21!s&}6yOa%Xem44@ z>3f#0FRi?L_M0n=hrEeatpyb$f{45C8eB@-Ra#l4Qg5bu_uX%G)ZQJK+mRy02c>)g zip=*COBRtwH~H1={jgBm8sJSMGGdv7?3Vs+PrL^8<>Gm zR4orrmM)tDRkHNqjSQ_i_>Y>!`ar7qJ^c0a>F-H$^!1YT3LXB^W`)O;Fek*}Lq8{@ zC~Zl67$(`HM!fVpZP5ZsA9e5TTvuDO_|&-;V`hU=UW@6s(E{qp1dv3|jRdW>=e9}? zufj|qc^*~Z6KQHlM3Zun!xr;{iZX|N+0RDZnDMGC%qo59@nHAI>>D~XP;MhQzZ{-( zaHD;>dKfoh^5w1$ihJYh#lO^nM|?mY)&-f!-v4--F!{y!4Dd`x&)!=GJc09{%r<$} zwm8ZSRh?(L-`Ulq+6*kbFc7^@c%__&|5S}-h`$p;g}B)U?k(f>=@Pq6LD8PZZSAg! z3k+iDJt!Hx$y`unwB+OjEB= zU20@xsX9k|_c!3r1IQ@Swm}xZWVxHR#SBadAdnFb?4f--Xnhe#wfw<5_v70B`dZP6 zbT-2`%&{5EG616afpPKG*!=E#4iG2{t>OR#w50Uj`^Kkr1xr^!QCvuh6afY^oey(5U|F$x}9JG;0=phyqzm&?`{ASPmmt^)| zZz*{L(4;;O)HwXQkiUMQZTHc9%ZkFSwH_1o4R-*uv6q5hIf_U69yhQDWcHULwYl@|6`UUd z3UF9)^9IS?X6|E~Tr2RSnXpg4U8`u|+$}Z6K_DyK%D1ko-lEzZ2+n>bb82m6zUjyh z1Ollwuc&@-dGHV**KG{uZOu7qTPlz&s)B$pYK@YwH?IXsNNdKy-c;D;?TT=Yi|_mG zB1OKr&u^t%)I8W*wL91fTy2NSbEuAm*aJ{S2zBNH$!}x8F$w@;8h$s;W_3|MhcFY! zk8*#(T_EYr^hV-$U0Tl?dlrMH#l`XqLr zB9CTWq$~sVakNDE#vz_wGPZdu;%VeazgYmJJnF|G5Mg;?wS#9fQ{keCw{g{F!Tp*Nx#j*BVtFsc;&Vudbd?>H6h zUB_+Njx@CE9CBa}N%LrEy&={>iH>2X20ICHUs; za4Pp87$3LMsaS?CY-wwIS@EN7bWI&MK9wFQ7k#Z)BHf@W*p3v9hvh6SvUfOCH7k~( zW{t)nt`OO1wNx05DY#@-PkwDPD|$j_@~M z2)b%x3bIc18FV_tW*O`!z}(gQJ5+szXPjU1j*4M2P#XhwR!opV-oBVk0F(&movBjL zb4z35205g8;slr_CUZGQ)v~p>znX*@HsBvE3{{O|?MDj0Zvo_J`4?Q&&&+lc*oYka zeBUvRXUAB(=>&rqz*Yl2iB`T2;V4Rg#~!HiyVz1Ku$a%Ng*9?(?^br@s($`C0Th)Y z*9^*L#deAZGXTx`*g~Hb1yQ6W{=gxbjNR(DC&3)!*gl>xNUwKO7_)@%-OZ5|XQucM zEaUO!pqlScR)6r9aqk`etZZ18HgNd<5~mXa)22Y>W3_(E6JWWosP@2WnfdUgw!Q7u zs5>{*&R|TtH+wZD2?oQgnn`WRj_LP2za-c<~Q-QK)=i- zW1Y(_XOt`dgAn}NdTS_M--n!Z!RA3s>?>ZJz%fwKOEy=^5VDgUdKs+u$KG3G6-cHi z8=o8@HjCo=xq;R1HcOZRMXw=;vu~bL!Zh(r<2||F@x9ef&UJZhoBf)HH8wllNqT{@ z?n$)$cK2W|-1A_(QJ4jv)uUSvijr)C{XI8_eUtKdnhT<3%ba3eNqWx@UO5-esAhC$ zLuQm83>1iB8W#YvGy$Y?HnQIEF6;FZDmgL772dd7q1Jy0#rVpYs2^rWOXgM9x8Eb* z8_JEP^|8S_Xnqy*HB4L|pfnX;hNr#Bo^j>P{a-)vqY5k!nkK^7e>WLDV3Y9LbJP8v^G&2gUTxkxb*Ra(I^f_aZn-4egP( z0_-gx&79**IW(lzKwC+)_t?b@B$SvUJnBY^rSzK=?(!omS;gHlsDiPl$mj;wWEH~x zvU^sUhm3$BKU-S_S}AxnWz@x6iJktBJ|wlE;<1Ncg{Qb%9FkGjtuR!T_q%w#^`?;f zpkCEJQ}%WRcYWPFe_k-k z*)gD^6iU(hG9|WD#aYSP@-tn2(ZFqBOW0d#$q@U><4#3$zBxt{dnT^fG?jM3pg9KP zg>BEX%o<^G?AL1IWVmd6lr$dh%@9SH?QI;>pFak)Wi<9N;b8#5V13pX%DDuKN&lJk$im*15wYgMjEb)_R zvrJhiEUR4`ASqexTyobo)hdjFVmB4@&W3Wkofa>Rcp%JPF7Pht{MTp<;~#~yIau^cfT zyzgQ%vojr8t8y>Gdn(^4e#WPYJ`{O1W#hi7J3``}iN_l~jn2f;HtL!z?wcRS`3CDv zs2w!XLGnwP0{OHy{1)iz35#rz9G|LrJ3=5U-wEy_wsVt-otpT1B{$?gVIJ)!kWR*) zxF{)pwf{LJ!vh)LB{(lNJlbEVio;-M^o@5piS7Vh?No_&D0`6|0SU=eBs=}^ODU9& z6*&}gTGo%E67Sm~BWTw*Uv4f7<4a-F(rd}OYe|)IJJ&lH9S4u&6@2UN6q@Zo;m1lB zClSn(xa!~?Hhf5Z>fl@N=?$#6$G%(=tcJXvJJr^fjV6i^hnD5!Gf37!v1n+QSc)~Y z)EalZ1D=Wmc)`!PWjS~4P3ZhWMFY)s+P5qwo7vG#;goo1jLs0I`Pg`VI5k?gj8a($ zOY>;^^b&UPO|}!6AD$8iI*CN;lP)4%{Q+r`h0*QU*r={?gK4Pi>K!?a?ENxkH{qXx zIWzr64STKf!N&&~pg#9v3@P*56~7~X+EIS-1+y;7Z{_u|UI-nt?lp#ZS8Y`U3rGX} zoYFF)%M@@sK{7P;QL?odT-PJ=`RxV$?cDVVoY5yH7OZkqQH?s@=r(4&?M-c~yOJS6ch&Dn@wJ{VLpLj?r}hffBpLsi zl{HO-#p$3c{BZ*LH7`eL3%&yag*CEC^&RC-e4c#`!N)-J-`xeYNrGLX*ofy^gYN2v zo1TiAh{t|za^uhuN^^`jMv!#mN!56m7?6_`QdZ<@=1z1tQ>Iv!p;_96>ilm=yf~#n zfl{Jk{icO^Vwd=oNpo-=_n<8Ohz)JlENfU%$f;-5s9JNj(fCqo4=J_>77j^-a6p94 zlN~FXH!4418odJMYGm7}Oirw>iX1&R9)I`@Z}dLIE}}Re+_sm5hH-|IkT~wulIKI* zx;zKPoqX24vw_O|pl0nd!@R3|<3x4TG6qgC)-@hP=d-t+FrVD1X!l0L(nr@AXN_yf zlaF5P49QpViiFpR{gizgzB6ijS<^qQiKfEv#$>;v`UW{9p#^yR`Khny#D)e~PCg z_+70W;=`myOAH64xxckDRSl`fU$5h$Faw!UMlF=jb8-#PubmCwewBQ%4?qiF~6DB0DwN z1{F(3lG<`j^YlWnT~P8LJAIaJE*)zk$e_B$bcW4$;puG zLw|CCxR2E4&hFIyp4=FZ1F|^yirMR5#SeuS3>agqrhMA?Q*aAk!&a+8$^sx(l-O-Z zYjvF|zCMj9woIi@NY>kKROQuL0(u&y`q-04&TZSGT6YFUy1O{3QcHiPEGHIaOty@c zD}8PaDBEsl8Lb-QC?$$`_ugX8lGW+4 z_QHJV);Uwe<)a3OUCxdYvEylv{9{?#Z~Q8TL)hW1*t_uS(Y zkQLX(FQ^}oT_hl#M4)onj()FamE#hqZbH2iY4^{j4jO4nJ^ncfI#yb60K zm)E;s(@<>$Iu+<=^-B%%(L$CZOClXI{3nyUB6}_tYa+z{5ZIZ{JW95`fI9}qao{>q zA+E8(+JPH$gLIyz4LR<|ZQqP?*}om_arDzJUGWSWFxhdj;E%+zCl18T?1rBg#y@EE zJ3$h_hkFECC^Uk?tq1Q{*FOz1B546?hdxNDBlz3gyHtI(qy)B7Wktk^ej_$`pG2ha zM~T#{{C^hP3yLFxrfQp2REt#333K8K%QO|qXI!(s8uJ;KbB(s}e{BbumOfX)eMI9$ z16{V-cRz@sjzDqKy4>ZR-bQXRS)<*Nlbso2$5i8Pads@086$8%B>@OqdbH>cddEJc zxr8hw+Z*A0q$7Fl%+4{A?e5Gc?l#3zCF0rCoY@ZV+PMAna($c;&JS^mB6zYeoHjM| zZEF`$r`%LKKV~dzIt`PwCbPw3)oGIH@(+?yc*`VaW9cL6o7aVi znv~UxS|{f_9l}BTB=gRo!UxV@f7}5r#1?8eoJsZq`Yn0qbs7? zgX?m>O?9wd+t8OU%ExKp@TLYH0hY=UTrBE3+mYy^HI{vmexHq>x#f+OIvLqoFyfNC zKu4%>lYm^0`yJ^B_g6$^lD zaInOs=hDKQ!Kw%*I4A+UH`#wY=&XAb#owqHBc_F!lI~L$#Wox^B@yeWu#4pvDpuWO*&o-X!w8KBWa?w@64?N50raX|+p z!5emuNp$1B*j#SPEiUB789u^ELIBW&XCiqiyt3t+$0Vol$yy{71m@!v_Oc7WHz&5< z&c8rq(G*UJ*yWXh7bsY+j_bRDJ0p^D0%8gd8snqb2e(ajlx+m8 z9gQC3ep8*_ge+ZEKRw+-rG&W-$>UBQ!$Q>9wU3`|Mu6$I6b&^7c)b8Na4`QqSSb^|!%!$oW;Rk>b_Q!tM@>G{ouP}mD=P)l>9=aQxq*nbEySlFGESau1<*X%PS|%`{Rnd zuu1UNPrOq^FT3c8)K1lb0&vjj{x)_Vr=_HFlZ$`kGz$Y{>^t-D}Dg&3>icH#4&uP0%|%U`2|4Z+1j6>-bSU z1w!v#Q_#a~dV4)^&{fiW-dk2U4A9^uu6}>(=DqkKupoB}YIv~SS!P9qyYDAT>m{K#Be>5O8apxEpk z8$C8k@vVHros_QTYf@?4h$^@`rP&4TR9Uh$Z-SJJPk^#GKR1`W6IqStjkW1`S$^Zm zvo0Gok5swNYE}zK)Uc_U)k`bB-7A&VPFsq@N8(staUFTBdAB{&usE*xtnPX$q*VHZ z4iJXgNtRU2Dv{rlVlxb61QQ<2s}D-DF1HDFmAm#;?!np5$J3H7PC*9-^uh+d$qS zlXYnw?Qg$T&){K2UGCq;$1AP$GS`r}$3U;QdFZ?>adLVSGs!PC8_P^S z)?}?0&a~VXvF^1XaUs1U(>mB@cvOZDf1#tt$?h0D+U}rAK%5=Ub%Amvu9^-)-*6!) zT9(W~^zn`$w|GtnmUet_6EOzFEK}0^0Ido)HywMzHO9|`6~iyjg^XSa%kS_lny^9X zih~Ug7@{p?8>c3G$0oj@)1)E7)1O^_?oSG%a>KXaKY>4E3Cw=v??|d2v+Opy)Hc~k*TF&bEGf}L33UvPXJ&L|l*hSNb z`(Lni`EoizfMqMPXsO`n_YyY-5|at&-P`dxNnPtO`MtH<55$<#wlW)p`Lo+^@iAN{ zdj};Sf6-+iNGIN*Fc?OYeyyIYGNzO@XmZ`u5^qa}pk0Vahzpl#&j-$sX!n@}sxd#O#uIxAo&b@Jq ze%f35O}sDg{C@VLg7bNs@1l+kpF?eh-IzqwDtdWv7YFZOjFKA3ugMTcxz)HtdaAM4 zu2}2Yucg3j%WLI!B&QtR7;HE8$|E3{{g+3I)H=JQ1X5n5I)YTZ_J3xFe-tC$!K|w0 zBs!+MS(q?*5{j?;l@uO)raOg7cCPcXGqTgp+ktq$?R+9V+ zupLUxHxNTTdgE*j$Uo=PduD+~$JaolYb2d#&+7eQmFykoRPD;92%)Xn!ion6uy6%Y z4-sp*X!%sL+(GTdGpaZGGghrErinvzJ*Pp0K>mu#LR(>582?6a(^g}QF#lWTJB4<$ z%`Fs%={wP;YcY=`CsusCpb;;L@w+MII?q#uOp{$}ay{x_iw42zOPiC}h!^Q=g$T{kb4(-5BD8dKk-R~HLT3a?wHh3^+#<$Rc8 zN|hkIV7p-3V&uqFlfqm7Sd%aP)ZLelvnMVl#}o=PMRoRYfD(Qpnxhua9Y`)+&3v7- zWbUsSzL!Ea`r4ZG8 z8FAdt=JdEhz4xGoX$&ymsKwC11B9y!I*Cx;OwE-T!J+B9e4??2kd5QwH-^dU&Q@YS zEtH@f9t)F$r+qa=D^m6+M{m|j>~=es^jrE;)UqQzuB>H*MSmmCzMxEY5l9XL(d$*Ew1q5 zad=zagm-g+lS2G+3ecYJf~-F9ANaZh9fu-n9}1qdzgGh@e}0oQv>jF{?TC*Yt#XYu zc#^ckHFlj%tTN8QPil&9-EJB|2}JwpSPsBOrR;#>(b= zQDg(F7Fb;Bt=cgP;4+lvE#%~^V67vXrCI=wfT7Q=Ndj7xda%sTCp|am=-6 z!O3xL@u|XthsybjZ`h~wDj>qd0vdSk8_>UbLFpqEF}h0gJ#?_#dmGwUJ^y_(Ss;DC zCmZ}{JM(JE&m7~dw5h?1yI4}E8+Ed_$rx1SJ(>8fQvr^Yvo)a0$bdB^m&j`xC<;j{ zSs+_*f5?=T3_&D+UY*v03b_u1j-6RFWD5=K&`;cF0+E%FsEYev$oD|CEAH4K;{d`( zD_`2T^-BwnmG!onLZy&)qdfu#12+40E8CrDtw6cL)iFz`s^d?bePLdYd&zrCv!EEE zBb_Hy!sa(5hrJ1KtIi0^wwEWxu}7Zx zqn0OH0bO}HN`XyMN!9qJUaOZuAGD1il%r`*c70EGjeP=3Y zTZ4WFlB%g4io*pQOhK%dfnFC=L4Z>dlM(CJPipBHZ7h$Ht59!)o8-9Lu7S=OC=z1# z(VF3NSSwvEkvJzzsLse;AQK#~T z855x|MSI*pZj>DrT|7)xb;s5)1Y6`Q&*vs-msX{6KiEbdZ6?xb&Dd5p3KJ^K6>?44 zPaBEW5&AhaT7KXa{95cCbWGtM1(%l_&^k2`OuE=L+AXd?CGWMHnA)L{(3={7%Wiyd zu>>!gXI1~Ou4PYSN_Wtr16%(m8w;LhC;YAbM&~CkvKc8Qbs~U9^QA9L9!%Hf-+j>L zlr@i>e=ePBCcx6I^-Y-*58+enEO6w`XrUZVdh#<4@F7lgw7(zuL)()q#M2GPbkvGR zeG6--89468P-S9$(H?dqUZ|%s{lh~x3guta2WAzjw{FsJy|bG@58W5=>e>VC<^sgj zDX{+gcO*h8W*TI=WEWrodaeN|V|G`p5j~U^iW}9C2pJGFi*P3``2GmkDn{q)*Ot9h zDQzkru8>@m>bi_QHEhPvh*6`4X_`_y+v)Brw<%CB%ny$f)sP0;c96y&9!l%tE zi^C~UreS1pw54m2tOXX$c6}R4EKzwv5~GY_hCvEifL%yv9`}q#T2BWn(2I1Se0wP} zQX8kEPJM$qR_&p&eU7iZh47;4{-d@wS)hE%#KSH>Z`Mi~-kt>mIL z0aHy;Dk^bVji`nF^is$b_qrj0IyLG~Kj^Swz7qYnu-|%y?E=|VXfdzvb1|ruG9DSBV-XkSe#H?SzXl7$S zIxO>Vo@gpF)-JAV*c{+Xw+R{A+JvhwcP-ozRyBYj;2B^^hGMSr%9!5-Oh`H2Adiot zUnC{wC3kOt?4OUjG}A9r%hd`!ageZTrgV@bj*ebzZVEBtdV5i~E^>Yf0eX|(u=ArR z;HWB|CQXDq7U=dkt1GQ{W+uOy5T??X73oELJnxQ6hj~PUzxjOxvU6aaXL1TsjHed`oX13spMCmC(ja$z%9T==!#E#0um|8N7hPlC?#lC0#{GIBVxH zV5~YcvJM7Rz|rUB`8$>Jp~S+r#raM5LZRG35ijp35*Zf(kJD5k|CIm%g^T9X@f?ob)_*^AEW}SIdep{`EI;J|Drv08V{P{x_gY9e_#xsyl=I75Q$-&##cG(yR_cC2Tr_GRL_lfl8r~~xkG4kkqSerh8w%B>vc0M0_0>e=RC4v^ z=8hoQtO%MU0(cCw+CK}idwGvdzT%;}Umcsm3AxL}YVAxPx^a$XP28&8IQ8?YkYnGf zvU8tHSsOvmCk&1TOg_B&RfOqT0Zm*MVIz)2`aZ-lKBV)01B5B^DG zB7uKsgCy~dDqh)ku`*dAZClvA@S18vw1b_-thbDCTR@fDuXrf*t2|)7ToHqB-uwk< zV^1(JRn%{{yFV_H0}RmAmSc(D>h%wQ@;5(JflaY+nH7)Yvw@MsEM!xQHD^P!*&~Y5u&0$^u$u8eN=LNdQ${$vMOfN zPpGyn9jfHF)Z;pHb~Wyma+Zo5Mu94-ax^$~!Qclx%%v*jc<#t3b+yb9mt&{x+ckAV zjgkP)uA6-2{EW}qFUuz>xjpFn>&vY`Z|QDo!0d-{DQJ!N{Po9Umv@e$B>BW8&mKDX z+G;6_I!=~%Wiv5+0UNJ+98Sq=5**zK@XXWbF zx)0|Js20YBY9Qf*DfGUh?wC*S(b3nsA*yvwp&`3~3}d12Ca?-Dp56J@VUmzM9&i%4 zB1VhsP4D{Tqf>$#FK8x)KUQaUXGN7Z-l3jtD&gYrF+$$ntAD$q1nzdDz!Us!qW5p3 z3Pw-pLZ7&!cS)^)m&|kseL!o~+#KR+cBZnqbCkz-NA};+H3etosQE}k;j}C3ma?%5G#hYV@kc4SXPO@O~+R2OD?UuJ%ie zfbi~6G!}k~W#^LHUXHs0*J7>z>duJF)QP?%^^CQ;6$p%;=mGfY+)8Yl!X^NTS)RA( z;PF#nDj?Fib_gL5e%MtE^PB}FV>zwW{c+9bOMYc6R4CuleIEw!lE4958KBI{|o#WzvDZ9QcIfgI8>G%T3G1&|&Vd zhlD@)7$54FD#cfx1PrwgN*>L0&#@dF&RTf_{?Nxd;HHm{e7=2Y-UYS44gI*asi1|2}@hYMFeA^RTqQYVN;(6MGc;2<*|eE}duxv-$M!$4#au)ML4Ny)(fk z>v*Ootd|tSA66}4>3#w(aNdq~F37IJx`cU(dSV$Zc*oP) zcYbTfy)n|+8>-_Na3SRib69hg?HOWloW{!4$_=tb$*drpe?z*UB|UKA{v2SI6rs0A z2uVIN!KOM7@Sp9FHOWXh7D4{fdH0+ty?x>44(s>WFdNelETl36Rb-j&Dkv@*(Pt;} zTPrGnXL|2$9#Od}_4f>>e_m&7lFepI&l5<6e4SdmE}lP3&3NSN)^6j?UMNMKQL376 zIM}yKBuWgYT{D%f_rik5UMV|Y+=!yvs;FNt@R+NP2mr=R<6|lK-KV+F&(eQg1|=Qu zJv|n3dD}UkRprEa378U3`0~~iBH@$^)H?Keu7|%BUFEgiGvFi=bnxW(aPF9Eymy7a zNt&me$FzP)-p@}cpcM=KkCzNCsy^A30EcJFcfA*YUjg@#a98_YMDR-?I4ig(2@j8^WQPpJ(3jQ0c7>#) zZkfxI#gq<^#s<4p+;j<&oCtJd_D}c*eI5@6tWvDe{T-@Xly1SeD30Depq0y;&K_Rd5%M3>@a1 z<7ew`-pH*VKZPoa`)yIU`AtKe-dK223w{I2I+pZ$?+LbBv_*6S7U*58VDxo9Bnjk$ zKEKpYaiyx5j>bW?of;D}`+&hq8wH_96-!Iz1D`argD8P=GS`YDbXc;|OJpz)K_~3Y zKSCMDX=%1WT*>L}s!)?|vi&)iA+iQEBC^^u_|uCkaR;%6>%3D7oPd%q+{7JMua z*p?R)haHlvjv~f|i}cK89gPAB>#Sh`?zPKT@cFGwryADY$w%DBv(87~9#>UNeCMD+ zx?es2)NiQM|2agaLoc;2E!apME==|nTpI++ygt98$U*n|+A4O3f{;K&TLG|dV{5at zZs=o~6v8A&j?(P^n9kEpa%p8ZdtwyRwt4@6oo?v)XPK5N&kUbd<3Gq&JM;2{{EF81c>~2=}*@Ir*v53d7p$k8BH-F-$wf<(t$ZY(B7jS7L#})2o|GG5KdBkD9 zfODVkaB>Sz4cZz4DESfty{Mr&y`HU;A69wEP7{9)+GAfMySDEZ>y#oe|6bZ#a_D)Y zLtA2uWu*)s=w#4lpQZb|AGs)_>Ez^UsgrkK=J89shncHvjvRV6HPRll`=^Op%MA%8 ziTreDKr>l;jb9nOB3IV!x(oF8@8ZwM)*X40T=KEtaGb-YtKN%#J$Fe4kbgE_LEu-W zcdJxV7s%$ClA!C2RmM#r0!B#GyYrWZ5wtfe*0_~m_XP3jDxgI8%qgNU^#p7}hZZ3v z`KfXR4F7B=C=7B6>UL9m$0SAXOPr5Bw%$oX<13-GYDt`W-SL%c^~i@MEw`Vl8~yO8 zfl^1`Ik~Jh*ulIXnJy!<=w~5%&?QVIgALr1{eW{>%1Ww~b1Knr6zUS^bWlar>p17? z_PHr-e=vKIJ)U`r2SM$O_)sM|R&B4r=IUSTW^KFYFp7jSC~$0kTpVK3v+-Z3n!4D} z@5~PNE#AWXn!6fLa{-I=mE(QQZ`b?jG75QqTP~!<$a{tmOi?}o8`E@>0QI_ioVS{~ z>t{55iZAoStM-a-7=<(RA=XFn7ocsP^7w@3^mgCNG`4@nOZ+k5EYh=XswsK*1)BmM zP|pWA)FgqKY}7I^j1p~m_U*%riB*8iTmrUkMfp)u{nkX39ipo;b}L2eMkdseXH;o= z6sT%QAAE~XvguYl<9AaLbwLj7{y&T3sn|~)`fFxoVXWEqe+MNgfX!G8Z%K#CdKj_Ht2PVf0<)6Rd+d$iBJ3<6; z+%eI6Z_H*V!tkT2kvmJU{3Y8*=zX*hyxHtuC8)*O8AvpDpQy}!tt4XDK8kQRh-8W?rk9T7oXX=g6?klad& zZR20vABqu2nMvNKM@5AoEgn=HGVLZ~w&A0weZY{(VMkLb&Bm>NY}k7nK{vYEsV(Mq z@0!i(^1)G$;I;@hfgCUR5AmVOj;|5cGtyPbB$8Wv`STQG?8a8jD6NpJ(V@(+C>FQh z?PF72mqAqyD|H$VG8v_+gliFLhls@!z|^1i?>&B$!bQU(QEsU}cS2SH>EnVdp&AVs z*Le@tOb+hut>tJJ9AoS;@Pcl8v}>g+`5{qF=Bd1~i;L)kT@R~2>+h{&m$Bt7$^jM+ zoMFVNu)&!?)NifK%*L~9z$GOpZybkzE2GaG)BUrzh-eydDHLi~4Lhrfzw||t@4}e= z%upoRQv}23)dWQpo%}L*VUL#x{ek8R`QopH6xANqNItU$c2U`J1P&*5 z1g{oeWbgk%V+fA-YEpJ~e1M2|I zBGSX~sc2*9kca*uAY2!={aJhMy{vCM9h{ZyyD%A6RzCv|nn9QOGlaIl?iIrB?*XHd z(!szr7#^>p<+xMxwCCd;dYUUl-1d)(#hA!!l?vU>K?K$V(_nxEYy}$c;{{v>2+20*&ul1~FJqsHJ zdtGVI_R9+ahCW0;@f2sp7Hu`$#0t&KChy*@(dpls5}-S|<2e6c^+Rcs(i3-8h`(+3 zX{!0>Ww7PEduNMfx-0=jIDM)u!l)(;n`5L`?1uPF82m(V!u4FZGU5hIZi=$XG^;Hu zQV^+CrJn+&m5E78n}gR+%_7PHjv+Of`AC7DE9RIcoTIw;3r8(>oxVJ^Lqov(OuQxs z6$@M(!S8zbZrnC^2yLOWQ!C4Qdnv(OH)A+TW6_xqK0g}|X@Up?@6+AnpQYXE9_u`SGe){`_C2Obi`s2A zBH6p~C9#)@v!vhW(hK)JiJ8m-X}YAS#8F{PW|DN|tl2=l;&Ji`=c&=VSy5L%2?`r^tI;VE?Nphgsu`RSO zBTCueKC#8^2l7;~rYq=MXxYxjd_vMf*4WlVl%pe;ie9m~5OzdPAn%u4s2K_XZ*ntw zKaO!XZ%jR|o3dyQ+iN~^OvI>p3fzDLn!0;kWns3#WCiR8q>GsL$>ojbQx&y`e))d) zyThj)*sHnEI)>E5%~d=neR2H+Oh)jzE4~N<$6;nP;HSWGlUo=HFSa&e1qXdoB`dd} z1S?RiO&4_?d&~%{>(C?58eY?mP_J5JW`m;DGo9tQyxgq)mMd2kJy_K&N1aC zU&tNLI7@;4K%kXSuJ(7Sa|W6AYGHt*lq7RTPc*?ku8IqI&gKHDt~+t7WpzjF+kX<*yi!n3Q#6{rqg5W@>`}EyK`e<+^7cef%RX4|bYY6&@Er zQTDe$qAz%Z)GxXv87dmTRs>#Pl_afWD!qV}d=J~sg1)dq^o+Gn<^os3=hs94bWf%8 zCu-5dt-UFwiCD|nPm3{(JB?dwBmDy&~`FO*?ElS6U#|(>PY@DE)?9Q;G@E zhzFe)+gWeq9$+9u@MiCX1DPCq6};3ho?(1WEY!`sr(+9Z-+EX91U)&83HS??(^qCf zae~k1jLujPZ4fOe)btpK9}-_+yO(f*$Ctv-Jii-VzQ#AjKcM)5nNUX4+pL;-R<06{ zxQE=8dvvwLym|gIRrb)+<}4j{0xmcmh$x8$;{)l( zB@QznV~|C#L88z65e6yTqs#sCE5+B0;O>3l!(P1->~OAQ)#IljEPY*$@!y`+ zms*9FjPqCHbrD-GP`@pwd~HS4`FQLJX&0YCxjikMU`nL^>(0|KwTiHmt9=nx*(O}^ zZ3oHruT95>CBZ=E$aXPtjYOtK3ezOLO;@y9jTBgou-a2VU6o zg6r^$>8{`RpD&CV7K^L2fjIsi;i)K1a0c7j)Wq4^gw$yIIjP(y@7{9@_c<@R{a}7w z7tUs(H`^}v`5i0U1jOw<72Q9HS-;v0vPlAP{U5R>0=@foVcY^)_SU^CkcYHgvN%np z^Ql))Mng8kpHdt@pTTN-K0bd^3GldxpSM68eR`fQRO5K4yLC5X$WDsZ?{p!4&UIrbHY^0}vN z4BcDXXOGD+!dDQvJHfikG9TX2L&Sj?Q(hwCR4~1?hMO;T)2Z7cIk%KwR`mTPR5y;~ zf&ui^0;FcEH+{A^hD+_uPO#C5DPRx3X})$uBL3q_K^}1xWBi?vOGHxkCwDmzHjCAy z_l_A&iv3rlt8n(7QCZZWJdq}P*jzAtIK5365A56UMO*GWiRuKv`gIh!%@3al!odv1gh1kcpz}9Z`PRF~JJ+kW5L`#`Y6VoCC_;ml&cs(H(-${FK7FxXE(p z&ivc)?hyV@Sf<4-`O4;R<_@#02n?0+r=A>P&j_C8WkCU7|HhouB$XO=vXI)?qW4MP zwF(NO=X@H2oB}F&n|AbQ#$1+l;D1^7FSZ#p@zcPcNmJ-t8zCAD02^`N@#kzsVnBIj z{f8#9BLRj9pci#-j6I@!Af2;N$fbbD0N0^l5X0Ei#~ru&4N3h9Q9<1np7#1z3hqyf zOAshQU`4@_LuSL!gL|}` zHoVNMD^{}m6Rf(UpG7RNR;Oy~EtGY&-^YiRk5ePB#Q-wJm(5h$2r4G|2+edj+bCox zDs2g7#%fn=Ivce|t&T;Ugn?rt^(KGqYemv&YN*GWl@WgJcf*0&-BwI-cf9x6p$Y%G z;XuTrfD2Q)ooLjk+312Gx%0}t6C8JRh63aFTohMZ*z>2kuA({r}dQN z;c3Bv@bFx;vt=Fha(+t>p_AZ4X_fL^q~lxVG^{AmEqAs#Op^n=NYlNWRq4gB48Nzb zHU~gWj6Ig++arcaneMJga{3LW#h*tjc|--oI+q-K(|DiGOVp8KBH@7dxV`)L+*Ce9 z>7~7=d^bE&cCGG;ldq#88@p1*761>q41u3mWo8`A5W!mTMAHTECvN3d9WC>08;rPQ z?S*?ud`J}IMM}u^JCBH|J@epb>z%lls#@Y?BiqR&iy2uYJ9$ph&#h+rS zOAo%zd8I(xqaeMlWVgd^ zmwF;#bBHOiKQ$fx`ZTou`Hs7vHztE4I6?3{z?xuswHBi_FsOHX=FGT8FaRsLF#WQ_ zq527|dl}TlhFnO1$;b@}9E)d5s5wzTs?N?bo&Pl?H8f43YUYIwFu*xk>$K=_Y zb``ZNDRl6}6TOa>7!Jyxv_xiobM{PJBW0XF0qOt(i1*g0u-($Pj<6Ra7N7i4_nzkw zwK=v+w82;mtoRdXDh*t6TNW3nFRr3f?P*i042bTQ%BAPMD3{DNu*ob(Y;-;(639KO3-c6aIy``83fF1AU%3kL%@BV;qJJdcmlcXxiH zV89tbhWNJD<4=FzXa1K=KkS2IESpm-XBsLjBC|8tMU+Fjj@Sl^MC;%SQr4yG5-iM) z-xb;X-!Svl7bO%0ch|X8Z6`qAVKm6y^01XLRRS=Nl!@ zv0kwOkr?~JMOSP)URMMzmrdQRoT#S|#^`D(s%+Mk_dxCjGU-5V2gDjT`unugV~M}R z-#xCvPLyD%#YC6-?c!!?ofyK%eAtC;ZK#%?D3aajTM3V7j@ZH{RGN|v_x(ZWUOa}Z zA!1-RyOnpVFNpr`T6iAUr$ih4HTE9@VSH6PlxD=~-ozh1^QDIZajIJ(X(6YTeg5y6 z@kmGss)1HU9qfYk=~WjqDz;f^eZ2Hjpg-mluP?DkqOTgkN_&nP>q35IgONo>Tz z@;9leo9>KUSo^XBldni}K!_<*9D1(+{-d zJGghNztcd`@Roa;Hrk(`57G3pafY03TX-PJ^O+`v+82dU>=ldN>m$~^`@agH`$#mo-1vMR zK2DhAyhW$_8|C#SF_Php;k(cJ>5AR2)_?{IiUF|)QVITgLE=Y?sZcT$K>rWyV~LP> zWmj)rP}D_ImO!kb&l#Vg%kbu)_Kbz<DGL-z5KPwC7#R3j$3&ZBzvK|TNK_o8GPc1`_BYc z?cC^QUcNjG)Eu3v=JLm(~(7WUocg{!H>3LT{=#C z3eFY5(N!D&`~*kwQ~%v>oS?MOW08C5cEhT0K`OU_cWS-NwY~Sw%?|T+xyeIKhGFt$ zMAY87r0+f7t@xD?ip=a1UbaZ4&0pWtN6jdq6uNpVi}6OpSLy<*8gv!u)`~pyxp$P0 zF3w1l+BG_o+k_Mktds`ZJV;@%c~3X;yyD>#kEu9+n}N55=lkX74l1ywi=CR(@R3V! z-pd9$yY8a*nvb^EvBC!FN2L-f^Em$A7C^-o>x|Rjs5ntza3W{?kdAo;j{vIn(|L0I zmp5WuVkVYuRtXUeVY8>#9(;Ivq4sW7WMY!w#g_(HwI*ibiHnp-xaY~|8X0_OTm?Wl zU$=2ri;TT_kV{n1ZivY=W7N*PMt-aC#Bh9A^CfyOI25ci$2C0o8TYq~@-G9fvB%>{ zJqL`=`*rp0t%}e{JtXH9_Kj+6OyV?_BVo9`&HqfQd;E*1UgJN0O$yM)WuLP9dxZEu z0VB{(o?O|*R9}XbZ4_;vScAGkn!G~W1=gt(shD7y(U=w6nysm}0Xn9% zk02oV`~vy>`Ro7p8xVo-Jzam4IZ^dnfD{@R1lkqL3r}6}{7|EH-4)0(ccBr&sIuVzkiz_#o?ZJ& zociFP@B)u7%NZkssG{x*EEZY&&?$&Yogu3rB5VX|qobP@|F|gZ&`ShRdA#3B8UM9T zDj63{n#We_F)FrZ9dX(Kd%a4kFD{TZ=8cLG<*`mjSD5pizLv zb<%fJ?VqQTXCBjX=@v#R5|Q*TEp)*CLcEs;EVohKi7F}}x*Bv16Gz4R9{@HxCqEG9 zH3gZO+s+{0g^@*^+H4r;{CtbY0GrNZLvoSZuKLt@kq#q`o8QJ&PL>3e2EPi-5a_IZ z4*ds9QKO4zzkR68T}h^MJ}-$&Q>g6m;OvJ`g7TsE`7dI!f3gdWsV~JUI$(Hp=F|c! z>0}R#6>uODor$9E3YPGDWE8f(z?kI<&c2ZjD32WYRYnT4{>INPr4!n81Y{F}yL=|H}&$ zhG4(fyIbeH{|^RzksJ^U>@{@$la@RiK)i@{91idJ%V2)xi2l20rzio|-#4mE>ff3F zzt7L#J|;j1EYb!omDMkdm4B_`KivfX^|zWDV0(z-`>y@!k^Ue4nWq9QQsDlOF5iDJ z=*wJScT9fF{W~-H|1@2`Q`f*ERWzaF{=pCUZv*}Bwl~3+{_ysGMbU;6k-cvj_`*{lF*Y$%pUajmUGhWuJQ4i z{~Y7l7khj!?mgeai|W&z(WzyRBTyL{%${68M2>Fo54>$m5L<%wY5T!)Wfvau`B&7+XSYgW( zt)V`1$}wDoASdjo`}K(;*%an>HUWyk1c3JY6vF_YELZ$17-3G4vx6Bd z2DYHXeT+sWMuk#BMI^2lIJ$L!#(X^ysA85d;Pos4+2p!Q2d!tl?L{PiWU zyiC5J$wdH0VE2j~6uQlLV-pZkO=YINa(Wd;LXgW~stGU}-%pMJIN$E#UAPzIn8N>r4;|rWoUpDNIrLoVxwD+{LrHW#DTU3C$2$T8% z!u+d7p1ax-(XIKaYoEqN@O;UWU_VHowMIJE4`!Xo3r|Mn5W^~&Bu2kD_jI{8^~L}7 zNc}nZO8UvYeZvQLiT-KbB7LM1s;rA0Ebgk`589}B)|0~i0vDS&ksC0}?1AufQ`6^g zWuAxn*d!EDIOzJ56A-BQV+6HCfOHG0Axra@ zccf{itKb2-{_q|IKtzu(m%3SU<9*>>$1sF2TqfiC3l@N6LTH}g^w64aG(D!UL;1IP z?~eB&Mxyw(ul&RKF;7cy3k$~#EXiVe#;^<{>+&{+p3n67YBulnKTpvs)(xQQDsRG) zqJ(~7hZHm)#)wrbOXS+vb=k^bRJ)VSW>2eiZlCd!wOfCWC|xIPmoTi!&ICM z6kcx(<#59s{%1ekUBkO#r&7htKAg`ahj`o#lA zmZ12m%OB4~(TIZ?kzlSjIH4DF#=e+HpV-t8(Xx}gc}m=tA=q5zzl*;TE0XVQLI{ui zyV=Biw(%2;>C`)W$Mw6_kdVfnnaZChjBq`oVA}w@G5{z$m+1m>(byf8*h%lrYg4Wd zz=&(uSvLB0j4m0bHI#H(6{hu+otb*A3cO4GZ1ZTCFV^{!jc$E3iB(L`;sVPf(cjGz zMH>!l*j%fgz(1@K`Z5;ny60NaU;t2(y6pja-sEdwT-FkREY}0sGTLIWIr7!jtr&c< zeZi^b#~3t!BdHRccJlN5!Z*{!wd9tI1~+qvF5{+5^GIK+w?Pp4{tG zkRPa^!R)1iXrX=}9pao{%YeAC0*fxYkYA6)P81M%gQ{YN8_YAAMW3_(c5VH!wrY4edIhW% z02kZjZIdrvOeg_(DWY{iBuQ{G1<1Qw^&v|xQ!MCnJtQZ+t1IaK#407Ay>aq?OspcfM; z17u^bq1+tkxq$iL4!n3^ksaX8AiH>tsgki695B&(Q07YoBFBD_dWMn|+XQVF4~WVl zyK?O2-9NgzU~DS5%VgeR8X#ckQ}S-E8dlgpiVrb@Y^hLggA$%E3~!1}H2W|;2teS{ zSN6dB?LQoZX3Rl&-f;c*r~Kot_N^tJB%TxzqpsDvcX+UVlTch4Jo6$QHn^d#U_=Qe zjP+n)E7~^_JOk!HapScxk*($qt~YaLf^xH%V37tR+Gmd`-FjGMQ$TsO`j@!`-y#;SQ}UF~mM=+BvdCMQt~<1T=sXA@kX>X-O>(SPtMq0?8S`iJHJ_07M1J?0JFOxUFE(c#0tKkj|j4NLV&j z;nmeEd}i$saFln`SE>P@a0A@-ftW_Xm_ua(dJ>pSHw2lqMswtkfqJpK)Sy?835biw zUci>>!PP=F?m82?(Lkz(0Q@q4kY2&v8n-Q9s{<{PZd?WI8hcF1j_BSH6UOIAyIm%> zhw-0)GbIYly&$F^PmWQw80I?mo$0$4FjNi7Q4pXKT#b)2o9nlrahTy0(;Ur9T?XI&4Jv<^X z&&*pVfSzG$*VcIUmdR-yK6k@jnt=yMXp03qp{ti3fr&5}0tt)Djm{73&(=*>UjiU2 zmG1lg5)kXckUzI9G%l}|feI#!TO~iiln3AjPHKQvn1F%*6ufo7E5?KFmZw~xn4u8B z7-PYKZf;CJ5G=UXV+M=|vbsNe3@kNzGtwcr_ybIz1O{a7BgfeHULuwVXQxHKS*Id)=KOjqQZ>Z=c!Cv!ECI#tWUq z$ZIH!Fq(9xbf&bal@^aq5)PyG(z$nEUagfIiQlA3Qr)IFr9A(b#1r1L*UpfGN3k8p z+GIt-84=VT9RgZ9?`B=dL>xUA1{+lxZoFQzNZO3m9-bkS>B=?MS!gZ8nTS>otjAcaG zltJZ{IPZ2M(*Ks%cTdqvwY2b&INM3m;VkGb5$buddX~cK7W!R7Y!xEuD!(VFe9Nak zc~zQ#Fq$L{aZ6wYWV}0`9HFT@<{shvwa(}ZM;c;Rwc0*-TC6<5pV%A}>5<FQ|c%k$-U1?ZZl*`v( zFG-^XyP#UuBf3Az-b%z$&eQV!o@ZY=HzjUpq>G`%239^&b_QT>KcvS$%e}A)wT8A)ui1e)ea= zv4e9k5E@KDS%*qawpG@G8Qb{;IieC17?EksRfC>@Gd<=|m{u=19kiITMRr5!XfKfe zmvOl77ZORB>Ct-B=~6{%W=pp85#H~C@{p@>1ZGRt_a(`r=vLM)t?Ma>OZeAi9AMu} zWo4K!1>Ys0^@izjq<5x4s|*DK8G9CFzY4e-$Q)>pRxd{BVkoJsC^JrHX&aChsB9}X z(Sqq(-se5xUS!9HP8TJ2_rIItClNQr@l$w~( z(cOi~nRXUPw~|vD!Mw!)^?{#JmcPKAI zW-)~N#r5{4j~Z^&dBqp!@;g7!BCiRXqcl#?xRH(qgrERr)45}=4^R1RCiQ*tBXsP%8^3F^JA%$Hm(^B{W&S`kH$s%d# z<4lp&S4$W_y;J=Bqm1PQzYd!vEjy)kg(!NTZ6^s(p#;OL z6b<{G&8t4yV)`MP?OJL#X}qU~b1IY<@aM3e*{0YG*}uGcMLmMnheVfM%qFXEX0vVhLy{bkW=B ziyZrKw6zHnzPBB<_sJ$|Rr&Df_rPlL8rIKO4}Y@dj8oh;`KB@cYJ%0Mwf`UDhL8&a z1yLXHYNg8?U^<2nM(FhEjQAP6f+0bkPx){o(BA8!P>>Upnf{7O9qI+_EYWs{#AyW_ z15+-rPTjZM{?4g?x_rVc#G;N@ocjB+fek;K=moTnLx#Yf^ z@I0zJ4>Myj<&c|Dd{edT=vH|xpJu8aB5Y+DIT$H6ElvfJvH8~AkDEy^uBObKALX6D zp1w@6{!LwCf|)@|idr%$RlG<$CroeHV2*>JSnW~-vrz9v(RQcyX{%{DnOvNH~t!Y@$a z=%;D%JAJg9GKVazpy$deGWh%i5QYf{u$$kws$Wn$Sib%c8%DTCV>=X7dGJzLTwzYz zMow!7R1tB#b|!9RpuW+H2u(YD0}rtYu?pc!YfjNF*Pww;-MPx>EVa>JSlL2~qF@?+ z;~RT>?lEWHj!;6={Nzrjey=;0j69_Wnj(#zYdjM6H? zjo79GO|t|8CvQP^ju;pS5ZqObB||sWjXF&TnDvFJl#yCLZ0fw5^I9V+CMR+UkpZu} zc!*r?hiP23@)tjuB60}zg$mewHCcT^(3q~OVe7bi15N3{qc#hx%oJko7=Gh%O zOGjF`CFvD{?*wO}tZD1Ld(pAWMg3h%FVBiAL1qwMXv&ZN=%p(<*!5ee;ycCo!N;{k z;ZfM%<7r<9l?##t(fwRj`{q&8$o2s*+2%lhddf)W?-L_6RLZPqxge$^#cv2nTm|VMS)IXVk32M z6u#C4MACoML`4!}8o$CHe=)f$Rz;pnEwD?>_F$hCkAcLr8-re|Ge2g#%sCfiR7;uf zN=lbKe+kZ5!H_H2T;U+^CLhs}aqH}HsIGp;w|;ZlegbYrRc9-g?33dufrfDzTZAZf zOgf%;s`nk$yFp{ah4CjIh)~GckU@9QV(ClKaQ$;kiNZ6!o;$+3PDhXk;%)Yb{)+O` zcKA=t7uB}g&3Uf~r+H;0Unw43Qr`$-Zw-i4l~sPl!B_n*KU5(zgcsJtFB6FT@IDY) zCO|Aa^$5}zw{#LrtbRgl9L8w@3JITn10=9GCN^V)jQ-0es*n~i>USCXK;qGN30 zqmjC}%(ER9M4+{y9~B(m%{i_aw};m%UG-9k%r?S`Hln)sPOsaSC)6JwZ|vHRvdszf zts@U&UAEb1UubZh2Ic#{qrYg8>!%-Bce z$9i?3IU3g2mg|RlKZ>)W;;IV5OfO!5%k=IFctVqJRW+L3Vbl%}?EBb(+M2Wz-c*-`vdwwVvO)! zUc@ixfNvNceWGMd8~k}maw3~;4ik^0ivI%&Vo61$?@)3=27MQbs{I^Dr;u)2G196Y z8goyB7M5l*3>Vb1mDIbwA3Ckiz7h;mZn#FBtTsBFjRULAsD?l zkLh2!`)Q=BkWPwI=F6*L9}Uh66UMKIBHnP9EvBHiFAXyNlqDO=s?>O&qo}kJx#PY| zBgr~r@hyInxM4SB^nfhIs>CG<$*N{}@F-j8?Hyv|wq;DDLQ*rziKE~5c@UYXJ}Kk# z#Je>V12%OrNEPAuATj?pkG z?|3Y)P$nJwl2Wfy_vc4Zrlu$8nDC>JZ0`2i7-?D@-n4rApjb zFl0VKF8&B9us&KblV%bNcBMEj+f^Lsv^F!Tr&B}n{+Si)=eaUriV(v zJ3k!tjm3c!yVU4xD<(hIK&A$^KNBu4Mh(zO^^cOw!nD3H=AIJ+8ES~zRRE=jr9zpg zRePQ&?|!7k#QJ2cq>^E%Nf6L%Za{Q{ZayM8eIA8sy5-mbq(C0FzU<{?(*Xy_NZp+a8)3%7+FRb)ehQ$?hf|CT}8K1 z2ToIJ+tvEu`y`>!dmlTL#R9~q;7FwK=A)GObYw*7=jhPxxD#kWJ#IcW=3Vjl< zsbG=&!bQr7+!HOh;!Ioind3uY84UgYWg6xu!TcOSF-mhGb8w`I#d<{^!(O`GWLwC+ z146uc9vOm*@`wuNlY0rILlu7;( zFWa2wt70jcwgYcK=xE4~_G&HNQTD+0y=k!3a3WFN$QBm{b2Efh4HgN&7cjt|5f8Ja z_*uJ%fu$Y#b((Sv+Tk0icUo?UUE1Nb%sQEW%jx?5EQUL(ws^$J3tX>tDLsEvYl=sp7g)hUGlRj4QP-GT*%;>h4-QI^Y>J%jE zgO0Wsss?{((@<-i|2@iWRekR6s*tA~Q!|)~ogJr<7#ZQ|Y&SaQTZ^MK^w_#8_3lw%$Pw*C#>o_Tjcz|PU+f4=sO7nPPOg#cSeOsdlaszqP7P{=5qJhJRTvM86Y?Yq{n*KU6wbynP zf>6iEFYLG`L=jM8#2jAoML%2wWrY%61QUOB?t+I*+K`W6M(5>*%%KObC-sVao7?*v zW)e4FMNe*GJlk;(wI)~j9Q&oTn1R=qvz@i`R(AN-aFkOF9yZn#3RE?u22RBv3o?>7 zm>k#$N6-_i2KhZSm3jXMo{%_I&B4IqGYsNNs^cr+FVNLlQkShI?qV?^n$i%VNE$~I z{sqRHr2vJ6i_2=o=`!vKgX)Hqjo9WCr!)1NcDHF$!{?`O>he}GD467AvW{Hs5w4Ov z?rBN{l`RLAIz4pRb52K;M)G7%n~mi1Q`8fgxK&{ZGE%bbsNezU5`BnCd+s)$wXt>M z#B*XI5$M(JRwnYgu%=+r7F3m-Q6}pNjR_MKpF)d#_5#7YBgqiU)pkjUUo0P#!^FKv zrHZercY!!~OTm%fWHthicvd~e8D4K}ZRf(d7985%cwjm{zGIVeGG7IIJoMt#1LDh{ zNo+cY&;}Yxv$fQI>smsLmtPT{A#0*aOI1TcKaid;-W_poxf(Vl-Ohr{v~VG*di{pd zy{v;S$+#-OG49rqrsG0q=&5h>_C`DD!Ym$D3ocdEx&!S>*sN9Qmus6mRvB3z?sf?r z3W?Q}`NM98Z11>xFui@7J0raIiav6ig6by#hOqn`=WSX)ji+*&kYk(kN6ZBsC6w|$ zzq3sT*VuyLyxyMg2%dh-XgPT3lxUP!&6J`-rIzK!$IojxFQpB7D%Ht0 zj#&2hx#!wW!tC-pGg%u74sflCG?dzvs-2!&&!Y)B4(``ITxDZ0^B0=oS&5rc4beQn znR$4FJ0GhWQ!Xs9TRX;uCPsUpeIw#$9LuvNdMn46#8`JzUcqussHBbg#VNzWW5&LMB*mNf)sHh)fY6IIGdVs7 zmL~hN?MFw_>%&}41YgAdcH4{jBcPjhc zqZ}X2X`Y5cOd)}#OX1MuICdg&shn5F_zp0xI@t^#LTpM8ynLm{r-EXR zaPOn>{VH-M1(MpFxMr$+N`7Wy)m;}v5=)PWjr^>^Ncf^ffAn(a`fM=!Mcx5d7)0_p z@eM{V8QD_Tl2%CA4jn(^MwZpg@;&L@_ZhD(9A(A$O~_~1rj;7}Ah`T+~_U}U;B{!(vdEx;mic&Yt$m&RH>HENsK-FrcJA}FQnMZAlbG;`okWCgO-05%&6i7TRDtkqy7LVVa! z128=G^)#W>m!gBbhV-3qT85jp_of6&*R{OqTDp+C8#5wPD5RuC!FOGS2_GV>>kShi zqieb@GGXXtF4D@BZ%VYXY#?I58M9YMq$j<_=x|%DAKu?}=;mR!@4gi0+ys5_ozBtW z&RWDb#qx@mo5HAXLp*`(g?)j>H4ZraIhS-@=7PkU4;l>w^fjZ3t)KriUTGUb)(EKW zXb%usf1O#+Dnop>{%|lwg5xh?y6O6K z2!$jgKVE~%%bfj&bJwd(dM6QqRx0}Oe~*r_@Ir{NA!fw)=`F#zf7bsjtAApvYwQKh zYN%!F@n*O;b?T0GW4)W6Or8|7_a;M}%&HDaMfwWVYfDNbn$2~#hke@-+QMMnk~;F7 zR*D8emw=3$3{b#l(9JYN`a`|mwboun=#}!2P?cbX5q@r{kC(5zEK@l2fTXza>y)=6 z#d~iHVVMhsE?l&^`Uoh|rXJVzNopLiFE%V()A8ToDa3N!1IUv5o8B_6NSo*CUazm~ zlQh=Tsw*otE6B9*30&#UPWfh8c^LAz?-bMX+pH&LdmFvwa`5ixE+d~uY7|bB>PSgc zMVSz;I{vhD!`9~py^vuDt+Eu1QTooq!YwBc=)-=QM3JxV^K$C8*@!p94hCGhvXe@~ zX}mewUNEovDx;HW@+mg)Lq&M9b@IKVIvQQla95jqcF$h0je-S&9`B zO0mv=73m3+NwIw@u1@p5>3K4f8^aOn^ov`Y_C+cqwCA^nrOu+I-F`45o+4!(H|+)7 zwDS~GJY>FqMyn(24ivKyi*E=Y$b@`WZqJ*IvOAbOQAJb~?uwuGiu+?^ERkZPsv-tO zue@4zbBeNpO8vh+I#k>}nK}x|gJM+_RwY;St;Y zTa}HYLx@Rey>B=uyq6Aq_m4wq&v-8%%6tU2rNA@XllKVjNMiA|y&34_AP{sIS^Z6yE&7aB!EIWt9S%v_ZqRM< z;2%M^jj2{+lHq?%2iFWS1Bk}DIJ@W<(h1K^h`sb%R9dNmuYwxO6#4-L>`;bK+9xWw zWNQm~U1=xN0P(()DULb9yaR@4C{y zB}-ONG|=z3xM?56H;uiurUjKrJaAPkkvg6?W$OI2=63f*;vyU5_I(@p5dn7J!|CFR zko>I#-Y{~%bxX!&>r5lPLQ3k1`oDGLgAz6TgfBj5kJV3%@mpl~-EaP4 zp+g*YbR#Gsk#L#Em9*;Ij2$99Miiq;aBVN!U17%jNS|0ovm*YowZ;fDertMH;3qB-(A>^nfn|`{E{JEM`k4?RoZcmz#q2?n);eeKF9MAQ=?YjTJZMA z4lZLW<9(amyS|pR2QnP5IqU2^a(4=YmUq3z<7+^%T(J7;`zVT)Smiio{?E}3<+GA> zrkam?8$`M$^kk5BMR>R7+9sqRvy^sO9G~$6R@P@MhrQnYcCA}`xk%IgZ7*To;!8^+ zvFO9jN-oKG@Di|q0%ccybYDYCSf=zcvt+_nXdcNbd>(m~Hxt7m&UC}^ zmeIn#fds)f53h`OI;SDexX8b5347k|`u6gNg14N6eioCXWac?xTIC;cd$0@R^SZ0C zeAlB-QXjh);H4Kz`fmCwl%^G115V6UUu(r?d7(6grq5fc9w`+}Q0qGbMXh|N{2-!b z@8cSd>&&d;3yAJ~n){Scnruat#4UQ{x;T}k5xf9qQBR0H{ZMLKb-91V5xO3e@~)st zYADeTVxcWo=R82LmeO0?n%ZJFB=xY+Q z!%u#MbM8nU*>GK*6F4$%SwHB|Vy%(8eJ0mTdP>N^v;+2YO34M#_Fh^iy;9(@cq; zJtTn_eZ^9jYFUTMM&sb|$m7Ja;i#uMw|2Ev!|hrYz_m2uf+L;Ao(cqVb*X1CbQOI% z3XAK=JE$YdQzmj1&G3g+o$Dd>heIQ(h*kh-&4qKq(DTs~_LY2G(W~%rs@BpxEpdLK z=#ktPw)X?VqfaoB#aAknf{La39*t1=4P|jf+H>SB^YYh4Gme{lnj$2Ni1-*9yjz!Rya8A?m%}Q((=X&;);(OQA zT=<#3v|Ki7CH1t=NS#}Fy0`7(5&J1BI`IMage_B3fqEfXY3wKVjc;mEZa7sT?mI24 zK~6dr@T+i^?N9$j*!>INif0Um-+O%0KE~GVR|CK^j|UZ62fE&r(4aR|g-B{ckb9FL zQ=#4c0edp)d;ukQU8bTNpO{%7&@rF7__=h&+h+ulc8bP62Ui)SIl7z)4rnB$6=Qur zVDS84lwh!pV$JV#67OH43}YN*^YO^>>~6 z34O9~XF9lv_p&7$MR+6A6WRBOWG8>gUkI^T{KTrP>ZsEofHRe$9}kH-f+f6p>xG9( zRyjC)7DPRon|Pokrc)P}j~INWCSxQkQEEN>6)XJ@U1p%o@s$4MPrzKFn-E?jy?BQT zebFgp(d2Hjxh$_-zFTB*2J65hmZ_J!)Z*s}7pyMMpBi+x-MHe-aVomGpS>dPU}GW8 z0%u-(<@0T6LLb&-xm=@vQ!Uj2h{UD-9)ekuP`H{~Sr7E;|8|ri#1cGBOhLqC4ore- zV>y5y;9mcVNQ{C>YLk62HSB)Y3RS(^8LXPAX34)M9Mz@xX0##T#rFN`0IE5d-7}qS z`WvPRNE~&Lk%_8gGP%e&okgfUdOXD^)~hA*wq#*<<^Yo>J81Q`+7~6)w-+;ojF45e zd?Su#_#ow~1A`1h@7=v7t+h>Imo$7}4`flBnx}5*5-R!vHdX zS;c4B5PXrq*3rj-G4xu$rgD|esVE*2NI{&!0yo=Y3}145=|N=pIN^!Qh9m?pZ!XkA zG*()zT;j{V4#N+(=D-Azv@^!Z&Z?k3g<|apAHAqfw;e$py-Pt2;yo39T7P0d^4LLC ztWH-)pvy@gtq|WdHjZDuL^NA}@FQ2;Swn5>L-p222>k4-uN`;Hmvd~%H1CrDYyrjvSm=rliu9?Saxlc9jhvQ{MpR*A0xgh7{ zuGsrb7xI<^QeRxNM#;FU4W_xwzbRwm<_Uc8)A(#$X|C|)!n%K%lNLAn=fkJRs9%E! zg%2_kSPnt2n||dKe)xnl-+>}ebrjp_mTp8XjNP6~=fuuuQGVC(tP~PNEDCMOro*AB z#72i&;h4LDW=0A2H^sm0Bv`pQ^xntw`tFw59_@XU=%uzUlLlM(2#1ZMhPs(2dhEFj zP)8i5w@#{}UvkpZS6e0`11%Or7+NVTr(_K0bq~T~@gcnP=b3=m+x9zfU+DYJwu0U0ojm*g{Sr1v=&}JJzNNwEo-&}5COCkWIz10h~!rK?& zGxnR(GC@3;{xZQSt(J6IdwElq!Bgy?PCEl7I^a%EogWRjR@ttN z6roPy*|W4w?5mH3{AA{0txy#8kgLQ0h`<6Aq#Hp>l$MfC5eex~5NU8B-AIEdoeD~)boWAOq+2=`-OYdAp0oFU zw)*VzUi;(wjZ4(E?t6|o=9pvrV!MOe7wX#QlTR8axx6uNzcYM)AoQ~}Qy#-9W2kdN z(#(D>zFx4Z8-4Ahs8%D87xXiu;ZO#KFMOk}5w57(P=3@|H$8%W+1s=~$%J8;c0OKWSZaLi4 zaZAg?r8|$*w7>+2q_qp1`gv`r{gdUUDvW!07TEMtB;SNG57~X6w3IDdx~>IXSbf8D z29y`Bi79u9mn6*Ht>_XJ)(Svx6X1=;e%z>SdP2s*-s6`CP}W4>syi=_gl)br2*)OY zNCrHhkr_b3x4gAszW{p?_gy2#+O#UimOqVaqn(d~?X-y1bE8hHRdN}SV{k{hXn$=R z!-*_mC#LpQ#bvv{z>nEQVef{}>H!>ToKTzz;xXsk#~=r`un8Mkx~EL>PHW{;o>;6H zfDw`Lz%BbkxLP)uEZHhDggb-V+})lkR9KE+Ec0z~=bMU|>U^}1!WL@pogb9SUEpW} z(wsMOEmeJ(H*!lHDq;;w9wzs!kPR>-Iok{Y{$~Tgp;Gvj)c}C?h}@#0SkcePd-zwt z$mW3nx5W$l$2b^ve(us@y*^ksEmRX|yTu>Lha2elje*ug*5?AtNtb(q9*v@c1LJ3JdJ+X6lPrr;Z9by>1ILdYjvdAJb={F8?Y@p;Rg`}VJ48!7KX z5mBQD)mPnEcE9QDsLbP@(Hl&p(~qzSsh*Di0WMG=BaF7GZvGTS_wS3MM*tTVmU#6o z>2$?dA2^Ys)n8O1^Iuf2kd`AX3kyttzTB5F6~x&|^r|mI$?rMmd%x`F;8lUmAw%z) zae9jYd_>+W!}z;dc%^>vs-PU~(Ia04rP^Ym>>fo2`4eRDud<#4G}z-EdZs%06RyP2QWJT4tjJS%?Qr>&bvf(<}@E;{04#>lihmvdPRgsj%470V`7bX9L*UN zK{J6;mD2gO&WBqY(9wYL;i17SV>n(S4X3P-;ndl3h` zB;-S5Dh%XC!TA7^C|U4w%l1fr%a+=2W&tn>&eA0$VP;3hi9=0~0$P)jxNp@K8kGSo zOgice`BlK5T#^VtU#bLeOq>nM?UkV zeQLd~Dm=FA*!n~`UW(hv4a{z)jeH^Zp*kP47ZCkzXB$G459Mm@fLjou=c6K6B+7EDm~&hJU&|v%a=w?rCxaJ)b7lfZ>JOq>ND-V4g1$sNMsy7 zO^#O67~IBu#fT$fDs45u+U0td;25&`B4z@BZA=w zD-K8EuKvviBuxkpH7U>)S9%87r!*I^mSS&CYz6nc^v~qKK_d!z+@pOoxZktx-}?1H z6|SYy^*o*r7@a3G#&yts=E_SH8tU2!{8_|j&HL|_Z?!T*)F9QB#{O;Jk|$j^nP|`o7=7% z2xp(e(8HXHPPt@Je)LF%-a z$Wi~2iszn>=rHLrlPzE>MZHJ6sAGwyfW;Krg9hCzH1_L}=HrMVsmVj&x=79~1HoA@ zu4on-$p_i-55Ss=E>Ra}E?z;K^t^=C0L>j~cHpZwCk^9`vqfiHhVp@oR`Vbz*r!iG zA;LHB#oL)00YQ}OQm-7*azWBZ=FN>>w`-H~o|`du^akC(eT!3RATAYj47js(svYZ| zw?8$^>AIFai%X`L-O^>N&ppG`V+N7+IOy7q)(nH24<3_+MjsP7*%z?h!t&>^n@2Jb zpDaoWU4+H&9xGl*kt75ic^utPG z&h4tBCI0N8rnd}SzMsq88Q}Ni1+XCvl#8r}6VppPzXQ_i>}lCI+-Q+Bmr!^9{qPb- zPkD2J8iIi{2k=Y)d5Vse={MFY8wJCmhO=#eEe1hnAGgt5r^)9BM=D96xxd!W>jB9c zFFnk$ygMrbTg-oL;Ih~hM#Pr4IzuUyP}&;5XOU}gy95c1t1xYtWR?@qLpPuu0SvG~ z6_@T7Jad}l%&UIi!Wql_9>ig6Ek=o9Nc!Cy!wjKQx{c>jkk#u*`N6PVMC)!iCe0RM^<_hZ3(R9s>q*!YtAa<1d*WHTFS0_c3M zUbVCo#%7kgtxvIcUn3*m3H9o3mMh@TATkZp27`RIaa*J2Z_Imu$#2Rg5(h~KjuS2s z^_KYO;6TyWVeO>3Ie$)1;rsvY6NigU zUmYfBP7}Un(xEXKvTiGAQ&U?=_q85qZ^czOA8j6d|18@G@iFYz?-fWGuPT8d!Mn-N zws@_P1J5u#h|l??cSOXd!S|SP=PiAY>g7eOhIHW=SN5vODtYK{fg_Pi=e)NCf@@Fn zh4PvrgNQLb)_+|WN)K!C4`*(^NW#m^%)jk5{|o3J-{Ja@pgy-8htQ8Y>9>J*r$unn zX66iZrTo#O6xuxI095~J&`IErt7z8J*IsvvTl92lB;Zj(Aj#>JA=teRv zoK94WTs+@#{KdIkpNbfkL$U(Ee`ytrZ`oF!x&rpe4H_2ec&(xrN|NS5iq!ccV!M!1 zk7?_N+=rjk;p3D9B347J$!yd5SNPcMoACgwNSqG2%BDgLqutH!oh zo?BdqE%=b;+s9m%MT&&0>A3kcGA%eQnj;_qKD_P8cN+X+$Bb#|UXL#1@%W@iQmz)OW8+;j8T;&po8v)Nb z$@uZPBj|l9)%!TSWvw2!Hmmn>h8n77B3~(P?9y#H5PTna+LsgxapdS`jpLEh!KGyx5Yxcq! zFt_+vLj@I zLLXtMxH0cTLM*wGn@U}bd9l#2M4rpaur>i3*Z}Anyc&5&v;@?-GmAfJ(68#iRpCC@ z^zV8ad$XQK`iTYlk$C8{dZ&)#T5EyO45r8mdV~?ioh%O{61d8zh!JHX1S2f}x(EEw zLVF(G1b(6fJnwJfJ@*$7csH%T@&j+cPiUi2z=n3$eI1Zn1M#x;L)?osH`!Mp%t-V; zP!p%=QsCmn{g}UFZ@mnV_3{snd8udtbcTvW(6QtGAPk9-d@Z#lVrKsk>myH1yMXYs zKCQQ7rh*tXGCkCIsoQ=>;A&Ad7v1OpJRfu~3i6CJ^;8?Fw1!HB>^y)W%B{k96YXk& zECRZ%a2Bt3fj>qbbog4oN8f%ft*83so8d*w?mU^WWV}oGEJx9XAttvS#b}e}J)Cpc z^NV_{y^p3jgSg8X-!Fc;ePknjK@mHeJDO8MRG@vx7O*3chx>qjj{_W$)Hy`06k-u@ z(^i3c^y>(Jgo?>&-4R?9H@v|#n|^>G+rh*ZnFT7pWOWr}8AQfE5;O6_>`598R&GDd zMw2lmZu-(r2k8x1)6h-7hOPSfE>(KTx7X4YYy%EIZp$U)96erZWw9>;4dpb@$g75A zm{)Jw1a^Bb+Z??2Qd|Hbv?6&Up+-UX?(cULBs1EjrkUcooYH^q%6RBg3KMX9i(+6y z{Be7)eQu1~2iz1d@b@?1U8y#oL#8$kW26NHXM(^Ljwt|$Y{ zfEx89n8%IjoOkKIadY6-XqICIBf-_CK42Cnq4!k#;t{t-M^aWEiXE>BpqrPt!l^gr z>%LLoVXZsp`1RotQ;NyjQQmMzJM)~Rd_$CjNwp6y_(VQcDkON~lnS58G1QiM(oBDd zYEs=5F%yCDH8FzWx}u?G>UQS3AEil>z%+F-aqE1~+|R2_VpQ%iXyznW3s93(%7(q_ zEpOaZAfd4NN)@Kd+A4l;7lN4`E&DShmPvanoZ`+>lBb4J)-2KYNaDp)iFzG^c&3=| zN}@38j4%IobK6-znDPtld@@^*I2y;U%qhFEyylO}BEr1zWwFnsl0#0Q!n(wjO8JEB zDu;Ez<(N?31O8-(qZ@(|$oOa6r(PVlKEly$m$}LD%|&aU(J0-fUVsh^2!zil{An_r z%a5|529WP&E{AJ*UsE2r;9;83Z7X17sbS2uKuOJV$g4==z6pvI4&hI+^JpjW9Ql(c zq}Kd+lD6RcPzErcLDMeFl6rqWw#l-Lfcwj!uBc4pjxABzbw)}dZ%!28i!j^Xqi%al zF|9FR0M2%&_`EEK&2oSQDyB4L+v+@~w9Cf~;0-NC_unKN<%ueA$j%;)DTOfOobkXw z(4#}tj9vP+zw%T3xwrzJE3Rg(jSCud0~zs+f$|dlO{p?ll44H4tHKP9n8{;S&Gs|B zIzTz}Jy5JNjq${xvUd`%APFGo;+#Y5fZv}=QK;Q|zmeiR9yNF28H+d|(6?Qt^Fp|7nUcaCr+V9Hlgri94mL8>eu5APLu=b*$hPXu>OL6G<3hf{SxnuwbwDY zr-b0a%9nz=r+~xKbIzEc_*Fa`b9du8q;!GgK`y?pVyj?B&I< zzOvj1?4dCDs$x^&$>ujd6`;?pD_(FF{Tj>_gUK)aP2JJ)%f!Qx|`?{emlr>S%XsCpjejrX^MMc)4Xv z^4ut)wIX0pYJhn3U3E~JpI^qsT@kr<=T41lXpAVdk&sD$qId?bEOv{MK+1L^sl@iU z{-*dh_V2BuZSw;Q;6e@8VMAtLh?xD)gk6TUBB<31=?zYsVhisnD7`+LxNXjL+d8J) zdK-_V(guxV;qIF+Ayh~AlSl4x7Z77k*=Qerw&CMNN{}I+4aoF2owOm*kQWA6iuh}g zJY7WAfZN@Sozic5d7GF%Ex}GBUU$1Z+S%XJ6>!%j@$TDRKaHrpw3ycZ?L;0{38y&v z2F(q{rydsInXvEfY5Wq=K)}Qb4*KTtMJ&H4$kg%B2shoe=uB z?-wc4pcO&|pHhLh^>5DGk6W?}d$*({;dxVi`HDaI{jBP7IpPqn9vctS0sFP+n75Sg z*`qC-qpQjEI(;d**D+q>#?E@iR2)%GztNYOl@;=!Ygh-o^1vaVnm4$P+4M~cVhW#J zaLKW`>fzS6MGfX7pibHfs2#jI>Z&g>x%OqDl zUmT7J>c?&%Ft0?*of2gouxU(HpO2vhBfqCV?}1$+81wQEr)#A13Z)f&q)P!%4H)% zvO%h^9r$utb5SFarch^%@furB;I;s&1|Kt~fQllHmx8rD_;aya-Y8k8pR*l*uSM^<53F{jf|fIGFWiy7E)$D_Qr&U_{To z@{G7znK`Q&qJkw*ni+~^eCIXXCBj2p4-ZsNXi+GY$rw8!J!0(H&8MjUCPfGH8R}UJTcXqHL;FZmG(+`7i%f$-^wwcIsa!A{W3-s{)FLMju_ ztR^^kZf=9BFC)jXa!-G3))Cp{DlKN+%ocM|^YfGa6H=rF@^YdLyH=dgi>!(Q+Tobm z1-oq*plUG{Z`*GT_TOte0I1|c7qgr#sDT8j zf^bLsgqyN3eC9h7G?}Ac#2VKo;CMl!1*I|2h<|;VoOoP*dUHc;Q7pP!21BcrA2S9o z+Th_5UIX+EhpfKQMTsdrR3>?><9CRQDF_VMIF4TM>FkFreo#13dfg#KCQ0D2=q2ek zY178SLg7E|AM7WHTZh|`t4%FRo^3C{6=usHOzB&9$>SVUGd*Fr83|-whh&;Re1BS? zlkGa!f|=5`3U5GQN_ie8=)gz^<{ogi+p6pA*3V~e3{dqCjN z7!1F6K7(j@L1hJWA($<0Dqj0xm0@ z_qyBKr}MYm2I@`meuWf3CT6Gx;Syo@;hJ}z-%&dA>ILN^=`T@?mR28i8pHI0KLnOY zE_&B}%1!F#@>9iIrc6w^9*@PjB0m_$;Hr5`GM(Awy_8+i%~NzdZT_&^CyP-VjROiJ z7*4hnjS6O_3x;0hT$bEho=W%Wbn{4Ph>sx`uwnJDyJ8pUnD|!R&);qFA-zb>>uYg8 zP5{vuaSxtQHpw4c1WSH2As86`85hyjy`LgxuAA2zVZvmzuDl1L%ScDM<8_Tu1|BbV)|=sKc~hJxi%DV_3@@3I zD-@1oTd$1Ty2;;JKDnl_+AwQ&d#i9OwaCu)rQ%!>Wj$-aHkcZ$-keMJgjf&3e&@)KiiaJU{Xri3OF7txhO%?D>DXhoG`JG@iN zSey0T2rIbcv?MvyW)!IEe=^i3>BOBQEy zsG-4~mU!J8S2^1p>|zmVs~i>A?}uJ?dz6-nlr{;{BO0me(1MYVgA(PI} zT~n?vK-8l?UFNvE%$NuiB3dAsz5#sx4DLI4{(4TzA*;(n8Q6YL3((>m%`b9Mzomqe zd&s)q??@?FY z=U=K7cJC;ANR}T37#D#U9~YS&$TpadSX=@o`EnjE-}DnBo>@#exG{s_lE@`vNJ@%V zH2b_Xi>=1TysEb|VML-v%j>~`bFZmw$F;@L$Gwmj}uiqK_rowOy#*S$@$6N4%$D@&rYk$%-SMhgh`+f?1vXeI(~z zLP?bg%CSQ+rFc~ih~AqSoU8JhkJ+|9UY8%qdFfP~$KLs=Ml^{gjN1i2Bxr8u>Eqrr zINQj9qJOjAisIpx!bCw@nFu%xpG6x=n6eY%32teWVzW*ybJ@Rq`GPVem7 zZl$aex8M?N>FhF$M10vKT#@orm=CeLy}(22LR*S?oI#^B7h~IdbhWneivMy~?3)6C zeeq@4(fWk0p|0p|X^L;hYH5fT9eCA}WRPUIBFQ7x?Q7;ufi+DU{H6TH-QQj#`CFw; z*-mIXZf8HV*SO^RSnbB{+))t{smS-dzq7_-8+rcyC?~h8F3vWRMf1ksjc;8hZ0gFE zI^!p&-!pX%@KO2D)RC)|mcolzFy_dhGtN#%(0^6l`O7n$Lx9uEOlA9 zZ2{da8AHL^&#v9y>|++@{zpUducfUp8QNK5om%!xXWaH4Aat;t*WSR;l*&9)t{Ns?{d_5G6YrtM1Q z?FFE+a~0BMX#@?Zvf(+g@amN>ZOQq@g%o@Tp7v!sQix>Jts#5Z!^W^JF7oTOw?Z7T zUla4TJgE`D_YZUF8Z{OulGQQR&?_2rqBZMmBV7%0?0>)Ih!f>DI~PK*-eVwH{J=Ku zyCqFCW5jypul1Ug9xdi8CQ6GNd?vJ}&xU_Z3hb#E_76(|A@df`%ZHXDU$j|i>K$lX z&00RjO-d${(huQ11>JrTIt4P7H^kPpTy|$Kt=eC`frZ_@XG|Wm3{F16e>bO^FdMG%gF7 zkA3kw?(TmX7VMcAkQ%Q?OXGsSqTuXc`^Ld-~*b!IA31?S8x610K$tQrF?ZXtqVV9g4m5-)o1bYpgv#uJwh)|40#iN)Sd($xN=vfcjdV zW-04W-Of!U$rKdS0FXoW_R67r%w0NCFcQ`WBc}e9)89t6|MqHxu>gQV6WE>q-M9MS zgf4c2zF`9x1WYq`ErG`HbWGEgZ1afc0=-+MpXj8;8Fe+2rs1cfYdccp;5=x+Jk1c1|XkB z1|fDJDPJETE?r2&u?gwG0-<6uFhDP}f)u+ll+!M6-TOa2nGAIQ$E~-S{&pp)JRsHF z6#%rc+!{{9C3@tV8W!ToLD3G3x=t$=_0jsNReZb2e5 z7b4UCqui?Wa(5YS9ztQLVcZ>0_b^G&ox(6w`qTNya1%>*5eer*OC*fLr@CfTH z>8JlWYX1D`(4W2#1_<(t>WNJM$~ym_KONf59LX8Wc5ELg}+~fx55{} zZX&7Mdh*`}(hfg(gwqMn5w`z!9WR63#O*$p@!tm0|34B!^w1BjzuS#<>69D0gSx)| zFkt}F$e8+-t0h=K?-&7zoJ;t)-1?vK-E(xHzH5E}>aqrK%#c7z!X;xiZTX;-@&$+3 z8_?OCu&xT~Nco$${eO61Je2+cFdSG=1G{0nONatDIjjVb%999fX+SVyYeNjYIe&>NfQVX1 zJJ9~S9I;3R;{SoSiRzzTsNNm8J)n6sLU4Ep7Q)#5IRuDv;Y4ZKNb^AbQJ_Z8c_Ygl zAnI+=K}B~BNMM`(WFNIqxRGv8LXz(ge>UIR z{*!$G&@+GF0TASDSIM)c*~DZ80umX+0Mue5J9j z7YHjO)2^X7mh-_?NFJeQtH^+e<}_BwzUaAV@f}q2DHljB`U|q+s>axdGtoyUn?De#bXM`MVp+kxmzS?)>xg`%8YWQ>Uv-Sr7ujoh)SntT?6s{EIkC zrBnyLcpHtn;Cqw|x)F+Vm*kwk;^^X00fy#q=9D5khsYE#~@vVu)%yK^MAak z93|F)f}fZ|Qz!FllRy5&Q0n!2RAO6Ua|z$KstP3BYj%L^17TJ%aD%<9Lys9mhmPT^ z3p)O2u3cl+kfrd?aU#&Xi_1JYJ6@Nh;z<4urG(y13!!|p%V?0h4C&WO zX8=eN}a zxD@&pJM}fMu!7e->%fM^rMH0Oe;FL$+3Fu>J)KfsP{*q3$ z%uhd7@K%y9*ylUA0*&q|FoMJe_vGN@g{jy|C>?HWOT%09EPD~dN!s@30Er+d1W&@DmA=OpCcVd4o6OC*gMiWijSbam+rRv;}) zPR&3E4>TEb$F+g3Q4H}`qL6z1$cV0^ne&qRXpn#ix{HDy;5#V2QY3f;+rej02r_5| z{Ycim48gp_d09Pt574!^3z7(f$NtL^K#)q&d(D$y3fQgq0_M$~ zJ<0id^R)Baum-5#J$CT8n$<<(Z53ekvjchqJFuF3W&d>o?<8_LALP|$KHQF&V1a%< zO57T7>YAe5^ZELx>Mr7}wC+8;&PyvO?*0NPkFUrz1e{8ydof~lP=|lomcV+64ivbh z0v+;yI(ve6(FKf7!j%>L^Ip#R(V75-4L$~Ut2LnbADXwUviBL7Q-oeZhqU*9M5F-+ z5WbCDA!462VU2##4wCFudi9>}2ZZ(J6YEgF!wNK$e8DluqXi5TIbpi;vZGEErp*stJ&@RD)M8(^?-9hXws7uYiUK)wyHm?`NUt>A3$b>D~ETH53+ z{OQ03m&iy^uTE^;`;*HHEjv9X;{oyej+dV$+}&MFU>`JR^YMwW(G94#;Z!&4?}qL8 z$A@1sM%puoGiC13`V6lp^4wUo0MqnmYA5q(~=;7y|d|6rxzE z1%U^Yihxt-r_{YYBVSu;Fno`#>(50qkt5BG}(WIvrM!8vbcbPFL2dq9d9%GUdXI|#J#gP73qB_i(M`hBwa*V{&2G+npY zSAHT4O^fp91P2PS_cFrlqwnJJt%C|qA1D$q#!J=k>33;N*jzZ`x4`{M^)OlRQqGzCakn4n*z@FQ#uBh3!#+W-(VQZkD>V-RX? zM6zn%g33`Eez?*m*bdhOGUK}v=#xcjK>N%i-Fv9zbuuG401zs%`{-rF<~t34c|HXW zhoCT`!vx7pWv)=Qyj0-K(NX~)f9u_Wk+ap5G8F|}GN6&(N!nLP;X{9<$b_o&dEGab z>wpWv{nPj+@CHl3SUz^8z4xgC|0=;AIDCi_tbtw&$rj@~mi8c_dJH7g!c+&4Ip-pQ z9xBMMp7EI%r`!t^;8O{;M|04~B?^>lmuR1dD0WW(UZ}+>;7AWr}J+HGssRe%-~pYExt@>j=}uvjh?_ znJFK7gWdNNIVUFgx!UF-$n!wenis#MtmvH6@;FBQbt)i*oV@5e-hd-tuUrw+2 z{6)6qN5M?Uc~UjM+n21W?TaTGdUpBVxRn#s$}3w6CAxnX!HxkvY3C>wChH zL92W};+a9$HLgGoJv;VeYX6S!?nCWNRCX~PKQ?${dI7a`Y;?S|a(y*0v2@v{)VA}u zrt7JORMPh9gNmBWki2NtJynYL+B5CDFG%OR@7pEeQ3|_XKtqRN;Qr%>_E7f=n{RVgXWEjm%+z(#R8CP~Q>=VG)->~izU?6dJf%a?eh81G91%L35mNow zI-i$oS(7`cn0^G(K+dFsgW%Gq_pPC>J1I<`c;?!Oo!_puJfN!eR_%?Qk;-2MJWAHu z!wyTCqgE~Wp6L}q^tKw9BA_DNvQ^wg()My+&Sefw!cSFsv0nsT$WW;e$MNo>4x zt*n3a*6blj`Ap#S*G5#`A5@H9gXHmf!S;-$=sMU1J4P)R@UfXTp5_%j`}S_@ebUJ% zXP}{$;Tg2M;1Hs;ryK;({JLa@yZgPwb7N(Iz?R8wDVo4(zQoYtw3@K{B?l5%m@Dd4 z$U>6GdHdq0n}^Vd3NBa~APFBEy-zz%p8*R5mZ-p!evKEfEz2zXO_zJ$W@0Qt+fyA^zf;?onEzkU}Lm~&)u>u9?#piVH!k%!R2eMYA zf4Yk@JL;;b1~ot{#72F$;W!@a(pkVJa`EcMRcg$p5IY z=qOk)3UOTF=s*B_2*teI&GLzz!d5QZkh5jA+Dwb^jMB+n(iE+@N{uR;3BV+)K%qXm z?Ey26>V*#8auefjlG|^1!(45@zMf(C)puM9{P&2&rOeO4#K5&QbsBTV{AZm0{K4bn z)lXjxJ=Hoss9t`HYbhGbnZp+=Q=wU67DI5>Y6r)V&1Z>#^$yBuF!`sh??pfM&nITh zDvkJ{L*C1h6cCBY_u4#VLUp^mx_bWguqr3%@Ja3@YA1=0Lkn& ztnINw>0050L}tMJ0WYu@$!g8rUEp-c&B9?29vZU<$=NH2>VhB1b+fCyy~5GTyIp%W zcpl3xb7?H5D0k+;@f|}yr-o)Q9=As*6qTxVF}GNdeKKe5Enam_I$WsKVVR`+N)hUf zFh+%~(%(CGJL@{n5U4pnT~{NFb=n;~5pel@?r3qyN`81J;`Mq3DtBvi@~osn@P3qu zMarRrXA^HN=bI-l$!DEq3HM;HAY}liClh0?g_Bhlu2H-2kdCuTAOV@lzWJ(u)OKtF zhLs$2c zr?Kek)yPNcp6|3Li=#S@ccN|KiqJsq-k5fH`&Qf27!yVJjF)6Ix(b*> z&8?rUt;;aYcHV<=FIGVpg)N-(4ICB3kR7`%*JyhMu_ZZl;WSfCup1^VgpROb0AEYH z6W6^>Vp+v8x4k6z)jDc-LR^K{-1bXjbr4!8+}7M8=9<^gJ~E8Z)%e4bV=L2?iqvwA*pn`N-@{IvG`HQ|n> z>h-l^h3)O^xU0)hO9b)0UM}zBF4vb6h!>%vqTqAj%A!(lS#vm_dE{g<5N)nIY!|XW zd49IvzZuWIMmf63jm%v)h~2QXxGP1tyvI9OtT1RyZ0f421&oOz)CRkPb9ZZ)lAfRM zSA;#+YADt9;dNFCVc7o1S@F-!ct-*aZMwNkEy(|ut$WalK`R_@HT#b4{rN3=6>Md^ zSM(o|-eqw38sS7{<5x}*S&X~-wRTcmwS2BXAR(bQk5h5+6&U)&d}B}5rvLbT*n;_E zFKowAMKeCP_W&4Bru4hUSPYBQcGbvTnq2W+(qOt!8XC99H#`AKb9FEs>OOnfxsLN- zr~3;yb>)W@8U`@Y)7FQJ+AEpu<6Yr8d734f`$}#DD2G%D$eoICdZjpFNl+1k&792;XCsX88>lKtj6%nvPDB%MCHc zqSYCftKPD(E-?U*;L+iW@>C@Cp0>xx*o(KzO}*NAsU&Tnj8WxPC>>+3>2ip=`W-W* zRI{Ox809=c+r4I3GqE50Fl4m-qr(i+dDn#hFX#CGuFVV$d*P%hui!SvArllzt+O$p^RA{Pnmru-7c43CO00W_1(X>Ov zo=L`y*g7r#SCRQ!kQDl*=n)rlP%6#=Trm@-ZdWSek_P^=Ca4sllqXvfhdU3k{e0{z zJwq{7pebU%V4NvYuIUQqK3((>xhe8SXs@DBwWGoZe#5RWAbH=0l)4EAzs)&zfq5tm9`)n4-|~S_`AV4YUp^LyO#HMI%$ePMG%}z5R6G% zL9L?GaUZShPMqy0E}2K=AEQ*b^NG?eZi1w1Cx*0Z)Td_7%Y7yr#+@|tB=A6?6H^3_P(C4f9se*pgy zbPQY=Rk$q1hKN@beU_el?KqP=DA#SPdTh>57`|OGZgZXNRkz3-xZCG+egu*aD%>w- zC4%F(i62&uSkL2*=H5s;Uazg~8~ieVx4fKcVHvbLG!IF1mn)`lm zn4ugw%hPUuGx-6a?W#1d>a>`J^lRZO^3H`X--?^LN43mDr=GZ5IS{PEq`wdgHEN~x zm!1R(%>;2~l01U9D+D6iZ6~-jb4|-jJ~cR0Hq#kuUSk<7g|&|gRo*R$=PaK91!9*z zxH2lgiRGPj2B^3vi72U(1Vfq@w-B#*EZ%*aGt=LrEKpZZ|18pGvuhONmlGNd{UN@T zlqJx2Qu$giY$6(sDL8-Z%2J>EWMZWsZY9IcB~t4B`lwiks?#y(bRgl|Ye_Q16JCJq zWVq2mdOk&Q_VFs7g7d*#AgmcefY(rEdxu>%$2?C2jr6$y=G=7A#~th&006FPQ8x7^ zQ)U75P1!R7y_4Y^vXtBI7q3U}j5}9JzhDn-XF(f=vEo-%f;x>U@TjsPuS)SPpP}>* zHWhLFX;a@MWe*a{wGI=U4tskHRi*E?OU`o1HAiQTP$t*5?oHfT~3#B)L2X$YqIexemeB0*LyhVw+JqxEr5uE&5~CmaA=m zn{9S1*Yaz6SAn|Ae&u9NLnVa9-POAC)^LS=P>WfvHpEsVgy)4exAPvK z0s)=$T#9n!Cq8@5A}w9r5Ri1G2fyVWUf+6(KDtz>rm_GGDEigZTFgpxLm%|d*y=RB zgo-2t{nVtLEDw)LneEt|mRZRWX|_H58I^}uMX8l%j@T~PdBh@sivP07{1dO+^Au19={eEG& z69g_S4P_%tzg}4ML@B8faz8xJ^rqj4z}2pO>4Hg{`cAJ_)i><;lArSPyM&KN(W+o; zaC(_&grElJqdw>H3A-B9zKhdgOikzC!L)Z@55@_A|r=1G}HMz%4q zTQm1`(YohLI47SB$@L?>_u!&(?p##et}G}J13D^mUEU<1l_+Gg>D^N3woplTEjA!! z6+}^K5Y5N@n3YD@twQI@=f|rTOiQAeN|tOvuOU6NYM_2<@2F$HxCiLZ>F!1ZG<9$# zNn{+w87{tM>soa@b5JOI1<*zKN%xN_b`T|TS4YY(&z>mskM8ilH>CF%AT=yd#H|{( zxn`%LX3om_elSDN3CP2%QH_J(jkK~vufh~!s2k0F6MJN z%zZbGlr0nJ4_0n28@?Ml_UUZ_?F-%{mKNts(o#*N21Q}{?GJV)tWjNCUw3a0cAMsw zr`l>_-e7Q^ApBr%Q97Yh7gd1X<}FWq*X4u7^T+A9i@1Itv~4aRAvhnRw!3Vhrty^ zspRC8;+*tD(=!3{-X50nphtuEA)LUp1a=>vOP)QG*w2)eGinf1X)cUwbTzHQmK?K$ zj#9NWh+Dt=kla=@gqLfk&uodm#X@wWmyG{uiJ)NIs8l+~=sLiteV1m45cqCn8ou0< zaXmiYq&NL~duKmmzYV@$$57>nS<7f~@SZW$!Tm;KUNR~Ks2fb7-hy#d+lL!p9m4Zn z$uVd4UShTaPnLrI`y0SUF}6rO*Bmo&f0QkwA@RpcTekrcY4$xp6v<%X985wyXkjsI z-<`@X?99w<<}DdbBv#CRY+gESQ$?>8ciPP?Fs3Egf|!rDWmu3Ne;mfsVl>}}BFc

8n%mVi2eOhS-hCgq(LM(4 z{=(b6U%Q52dZR5J;g=*&M9!)<0i^qNClzp;jZfl=bcQZv9;P<)v;^upT|5gkFsFZf@qDdRH#VkcD!YQlz2}GB1$~h1Czxr!4mW4Ue=ovFuPOj zmm3|G_t^yeac9-RJW>kYJt*bATSl!$n%Xo}uSOQ6!CxJZnX(On2()M@Ta6@8neThTMdlv3sM-qXmAy~-Lz zcE)}TWB7&aV~y-Ch-l6W@Lc>4J+uuKcKe=ScI2sAmcpznGZjmsNScpfM1_+hEFMiF zjUQIc?@y+AnydM&z*uxa6F#xUqB_|7R%OE*%zJj`r8<_ZT~NuUm|{dYF}xFYVSKqi zD~qYf3l?YGJLt?d+D7wnTjT*`B*Y{0pAanm(45$i!|3Cbrk(JKe=6jDl^X138*%hzK_GA+*Q0At@D}4aamn6cO#th~|%rqv>29^{@r;x@Zt@%U3X? z)ID83oZrJgYUhaRHelMeD<(y0tCZkRC=eA6I%G>Wn*BjmmTWMM;pIQvmk1lG?<4H z3Y?vfJ1L(7Ef8iLp_7ep+l)lxkp3sqotqJ>e<*oV@Ng-~gDZ7@iG2_AN0cHlFn?u4 z*)Y68?jVbdQsMqE;8&Tq=2a*X-H8yuUzXV@v%p+^#;^S`*PCZsyONGF^1^1x2)t=E zvpt60kn>JuwDSmBt`E5;lfwLv{6V(1G!NCD^8hnOydnSO;M5n#Y*6G{5z2IG-AHEf zR_g4s&grYhCHGn>g_%?`CGBC|bptGGatYdvIMkw< zW{H~b9gyZK(XG+D5Fbj{Xu1j%HXKU0Rr^N2a!jg-@aiwNQ-PxI^l6@uLf7HbxXdz4 zyi(1OQOHd9>SRf+rrf$R-WgDO$h6pd5z1^UWzVx6ZQiySx1Bv%u1Yae{*)@hUzrSa z^x4=To24 zexoYX`1cgpvx#fu+&T0+o_MAB2*HKhYjQ%po{-3wIY+aoa4U;cg0k>2P`THIF z@jbqW?XGD3Pnuy@#cl+~m=aP8KfwYTTLsyCIcP72-wA;_no)sqc~P&^={z>^CL2nd zp(+FC9H{4n8BRC$ilgfzN@wdkg%mRKdsvmoU2fT40uxj5VOLLgna5;F_a<5GVHoaH zPwHO)4H$ESM#qUTRx3Rp|Jji?*gLR#2is@E!8QU`t>4#+ESf0*=nXeO4$9%k;o1

=5qOU_mY@D;PZz zYe`JrPJ&9c?F5eXh4({J-IPHOPeTy2{9E5x$*s=~r-AA_vg#&ois;=m z^+Me~Kh18`;T|PX&KVUh2xjs|yq!1Q;0e15i`=rP*bddIBjfe&I{CessIfcoX)*SR z9o5&EAi_%!?>3Q?CXi=hE*ORl`(v8ObG{Ku`zWWfIO~K5NW7weAGkg#pt-;jGrF5< z!{*ipdf5hP-Ka%=SfdmXltkyUoxOQMytA#C(op{qY!fsktA6G`RzA}_vRAdyH4VLN z{2frLg0;)t0F|MK!OyC|+jj`3oF4jXb(B(h1vtf!s%);+|Dc|E598r-;=@<#%U@1# z8zkO9>wDpBaIxDV4vs91+RX7dOIpv?WRb>2kn$U@$N82k>YG)d5tPtRvwrxGT%S}S ztbYSQN9!82CF8FTo^IU#0k6MJd$VfY?fgU431S}=1QR6&=KOn8Oz+*yR5fGN(@2WA z4k?XuIQ;-E57L}xLW9#5^boY(3WW?dHT3A+t?3JN#Q`_#tZKTXaX9dmsnh6?eD_2%#0TT(E$zunqmvd6OiNZqwWja6H6? zC_L1P@_hIt@n2`vV7c;c86axV$j74cWfH+5`Obbgk5{CMD}-;UU4d_jlwx&x1&?4TLMiD-+lmY@ARbF?s#Sj%|LrMez`$ zw-MB}Y1-~YdNMf~ZT)#?JLH+I?`&;$0mt`ll zS0^_wj^?@%>&4zD_8Uomn*|}9;Xq&TI|$D0Vci8$1N>-(P0;X5 z${F086Q~y=Puwv)hs!qNBg#i!71T=AKW9xi!fC+s4>_%!9bc~SIf3p)NC2d z2x(>RikEsLx3XW zPlGG7pniaOKU;=>F|y7TJ@U~QNZ>r;y%SBV>MPVv`iw3FAvO*4`rnIvi(3~UnF>KL z_Ts_P`yEOWHxZZR3n}V9 zxI?uj)wuQbP6bL8TAYWxS^F4wvo>OCZ&55FMgg%?ut_$nXXhj+SXG#BsO6z7p9lfe zOxf(56?0woV@IAh?hBd+i9`dx8wdFXSZZqOb7ys6yPoC0##=_h``)v{2{*8c3#EX( z7pOG)Ij}^|&vaQn;w;yv9M?vci0F52jkfM_P}-LY4d!U5b>69c68L_A?G}8wpScM3*%oMFp&I~Z!P8WH zANXSm<-0&wn{(#FTE#@R!gWYxPHF5r_Zg0vHL%>`t)!tJfj3s;l=}V9zpGxM2dd2b#Ib*TjZxkd+21W&ran!?g zMl^prNx8)N%o9l~UF&D?YK*djno5WOoV8htL?U9#7-E~T=#W-S&m2yy#Sy)tsut#^ zU1aLyeTUI)E1A#Rrv}#c8BERrN}3lZ%26$GC$E#n(!h2HF61eHA{qDbUc zQ5XcwZs0aFyDCt3VDC+vqDAUO>iG=cSFC+By5`$Bs^;nw7tTEng03w?c3G=sJ1B)E zv-?H;2evF(Qx?0-K};ZhX&OKcWh#&?6>JU?#wQlhFbqqzu~b_)_2P9|f@5LLl>qq< zg^pt#vCci10_nTO!0bMAajdc&`T{_@xg*{VQL^g?gSn|mL#UpSk(x1l;)GSJ!LSFw zJvC_s-aI%2tX;gnPBr@VIVTHNgW zEW5G0<~WMoV*E3{GZ@r9wg%4))){>2r&}^N3J!TT;9t9DU5DnJu@E$_Evbuo=}`ek zujd$TZaUQ$p~iQMBlA|DQ%UA^5XlmeS*Q`^Ktcm2HNa`N#FspiMoX@HFl`?=<=skR*kOXG^ z47`ge%JJ{2o~V7o2LrUJd#}uUVDSuNhC`GO_N=XtFK1QG9bBX6tPdrh9KT&TJ?>>#9%rY(m;yl}G#D zAW&kNnP}qI@eHfm7juk|V|aUT&drAb8WXV)W~AP=rC{`QKp_ZR;}q91AdluA zef(!X(4eddo#kAp->nLz1^1ljCX%W8gN{NC<%RhG9bA8y++19w|9ni*+IlDtSfXht zXw82%TDe>DAAc$g2=RF&=aGt0e%jE@CjA6V*0X zpb(J80^xm4b+!ER3c{5-=2#aKB|Z!Q#zG{&&-bUqYAzjzz&5If)A+?J1zF9Gflg`c zK4`=R6*(T=y7TiW4)-^`&8E>S$$1J&&`i?Rf zo-2&RdW`w{Pr=Gu-m2rpLyCXZQq`V)b(S*F2kFcFk-sh#^y;&z00i5TNi_wrGK{DP zLloD}n{SX^U(a7LW_6TXm9V}@v)CfA{=2DA_2IUM1c~#Rj}j)W4TZ}sR0lpoReWWD zE64lphy(rxs31y6cF6TyZICS zFK(AvUvIX>uOLa9t=JL-YzD_q*4d(p*hMl53xPP}&UwWIA-V%XdTuKL(r=-;cC?Ol%<4MnaJ* z80(SHBO!<&2(jWfL1lqI;kC7lgBVm1#mIB8a2Wv zA5Soe>%J0uFMjTZ!rE&r0Z=A94&SxL3j}Q#5ks+Up0&@!LhFc9UZMRFFK@71g$8#j zv|=!L>=xWmT&8^DU8@u>3;RM)=*&VYLxgym`srG?+gJ~9DMB7t)L^3{N+}Y}C8#Z5 z==k(*=T9KqCn(fo3-gWa%*a%Z7ruiWDLiv)*6pLe_7&;%gRF{)H%O?wLoV?{PQXPZ zcs|Okjjwbd93E%@X!i{ErDWi@lh3nx)gF#fEv=PuB!{^KdAkcpPhC7T@&ckmPZrEn z3G{X2iup8z0gPn`zV z+MUazML8+r_<=C}8O={fl|ub6x)5Usu5YFDN&eNR;2da?h1ll#At6SwZ_1gDFuOMW zq;6n{v+g>SHoKGPY78*AQJqnAld6ww z_vhIYWji9SBF#y>j)rpDYVdh4xZ$2dGI7XmwvgJx>q}=T9dWE*Dl~HDK5b(KHJcQf z0#?1BAh@ngY_r4{nP(nPU6lQSes`xCCWNof)(t1!8tFxhxjy7wPgHry;{yRAKlrJ^@%c_w1mlWM zuiDnFQgTHq9UW}XNnjCeJghWr{{);d!6;{E69N{!O)46uaH{Q_$gLz~gJXt+=)TV2 zdDu9y*lbkdRS!RB-h>^SFP;m2-S&MR`#0^+1Z%MvLjXoc^k$|S`Q z2$ks%y64I7lV^Q1B1dpH{p@eJvq!);eeYh(LO?!i6QAvsWo!u2as>=|LUbo~|0E0S${ z2(NNkQ4q=T$k#ce@ZD_NMP(Gs?=ApTa~4arbPp%n5unm?g!62JM5=P_hdb%8@gg%8 z;uI)^h4z>WLUe7PhqA_>hNeN(2_7G70GDa+rC7@Od-*=qRv44tn~0nBbd zRBBfRrKHEESM(01;aI7AfUG2f=zZy2CzOvlt4Rwui|w*>|9Xyy1~B3*zKx{YPtJV1 z7_iO!2WlFo+!pUwU?${eejr2XJO#17yimmw;q~AFY?-{2Oz+FW&F~^S@F!ssw8V9{XO$PXef+Gi=v zHaoA4kHE324QMeqDB&UN^XOt??es&pp36L5vYgo<)n@?@MJ5oQ$4KJ(8OKS3Pr3Y*>Z02_`cy_VXOCGqSJU!+D+$h z-A>=u^8xAdS!mrF1(?Rm;c^X3dDKjl4LceCmXXbrj8c=QJrXUh8n zV+-+sx_7a^ElCkMn`BRcf`#d2D&COxvTqKU5eatxy%>T@qIyChi3P$w9YUzo6 z%hvt$6oeYveln&dt7<%HQ>i-NCw$b6<*k`#-j}C89g+Y{PbMswa%{siy{gWfz&DfM z_^L0mWd5+-nkPDG0@-GVT~6d(3;rxd78&?x#FctTYwWv znen*ym`!aKnJsTSAlY|tbl#aUR*@q%VFBVe;?K;v);C$-4C?!5jTFw>;K&w3pUL^# zL=N5QKy6I0d)9dkdFYt_W!Y~K2RXYsw2;G~5-naGvmza-e^k3)iY5gEOF((D0;WL; z@Y)$ob^BefDB+@Y-5{WoJye`B^+tCTY|-6xvoQ~m`vU?QKvXtGn&QL#1#pK!{4Q#rh z#g($?PTzy;6f6bf;e{Oz3VqYYw7o56>2$@qR`>!Z;}sMI5qd`AvlHnKdPh z<7gJWmGtR`P(iMuM6zU$>}sJ}7xaMp;^yN9_ql{o_ph;$+D~=HZ|pfy5125syR@qu zA?EF~cTNQ5#6JuuwWfj0(ttZy8XGQZb@>On5A8J3rA~alzqS zTcO4g(#4?S<<;8g?Rxia!1n0WzEC*yNJ)q=k^rl~4?Ycuy<)?X0-(;;fLj-4K$JQ4 zk+oPiz!RWtvq{E@x5^vUJbjV_o(3piR4?mWu%0$w!cWEK`Gdf&O-!Nv#0o2AL<)b) z)Vx6^Vvs9?H5SHCl+TXX>E4)kWj=29M7liD<8-c+hOj#()@mPjKE-N3rN&#;pNtvn zI#B08ePQc+@J`b8et+L;DYAK?Lfnu9*|admzSpdXJM8@^8LYp|ECbcfwnCV)wIy-k zlj~DHS=}Q_h_<>q_d4~Cc@KDJpwrSt_^{Fdk)P&q=VaaGrcP*66$aJT@wsHCM>Zus zFxHj))JmkSit8dy^^EOKnPClSi3?NFw<1V>j%fk?GXj5I_{89*(#EIiHjYkJWC19<=WA3;~QY~CK_%lQMbrZ%-kb$)rD z%IzCK6!9oWe2w<3c8^S-lDZp2jI)AY)PDP0YeJfR?3=f9?l3i6{SUwYl=HxAD37ST zg#X{3@ysspjQa~wYoglUetWnHX!*DLsWX22Tg+z<9{=HC@Qi02%-1GZYku{Y=iw~? zaM8E(cUS&FjDGWXTd#s=eDn>o|CfvW-*5f69hi%_IXqP0@S`x-ps_B@Ys zOakz~ozC@EP+(5&{cCBB^8Dt#whJFW3!ZV`O-kth{yM+Bd*xdo#23mp&8&?$e)+96 zA9?)|c*Zl;?cBc?zg4I6N6}W_;2%X}HcWpMjrly(|0vojGxW!!t#T%RJQ}l?`Qy=; z&tqNC|I4A{x4=gM!>gi@@R&+KJ!%dpF-*bE!iaZ(f+z-{jq)Ir%WXjZ#9yxr6&l`w zGR`zWn#O_l&zrr_M$39Aods+fim6im0CxcuCJsYP={|sCr~%~bhS<3RbPOojmyd!) zLv2YM0N6MUjigV^se)Fkof4>MAL}G(n%3^gpOHgISb2krgeXp&prb|`E3%YD1L7BL zCA5W=FQ|#sgB8T}fL!Mq)K-dATUl_c-f>~8W8eJVIQ;Lz%*GH9=lLJ*+Q<|Q`KPh& z)D8eG#RiWKb3==O*|ur0Awr)@166$^kVB%MZJ#*a1+bGQP}$>|Fsx_i3f+q_-WLOi zKjI)*$VxF)VPFU#v<+|5mu3&$_>P)`fN!_kkOssMwdETM-=Ca8y0~_@LFL_&frotc z0DpnOjxQRslryH(R_fe9S82w~y3q2L1q5A|MYATBcFA^_>1ittzp=A`7B2ZUWtBV4 z(V@QK7>%`v62JhxP<>heASkd!52`S|5p(nsl*a|Kgi>?sN~VZ5_X$g8bE1pg={Ia! z0%^=4#h0uMaKaA* zj$K$MINbVEn*{XH?NkLqgB7L_4Cu!KvRe;Kjr9y2+O-u3hH|eNyrOWQcnE-m#cv?a z{Wl360LCRPg$97&w0ED%bZ?Yo-R{e%nvRg`HHwdL>+_1h+b|vdude_uj${gQ9qc*B zD5mheR$KTK0lt(07lV+vr{AWgR6=48x-PV7P{_gg9%d8DSaK2L?T~mi04?bGDQ;j7 znD6Wqq;Nl}G~{VuF83Nj1r)i6fS~UfpeU<+KhTd@naQ{fpdpY-XO|B9uXJ@_PdXC& znz&7ctSvHCmjU=r+aKDy8V9F3uV{gZS3MD&xIu8fg9;kz;zMP*+h9Ae9w{bw0TNV$ zB<)<2wSyD?$+dDNHXrTWEn*wVd=f7am4L-bAMkpp?GWq*yR7RWrjwuZZZVFu9HqWK zM}W9tUstg-lGTANH5-j!rmT_nEEQVnw4Q4I5HM*1`Vr*b3l?vbMwg zEO5E*6+SAU;czG>f->n+cC6(4GdF1b^3+I4Bw>m9@tJ-6?o=S8H@n4a&oce|4EM!v zbD(?u_8tlvDqBP|W-DU#ziAPB7dKZlS)a&EO5E-&HlChVKLOAbibJk%+@=6!o+t*4 zW^HM>B%0i6e8d}=4oNUZalRAuCPiZVz+?rWB5J6n)lBbSKS3%x2B8yoC09ZdlLRLs z5PO6S8d_)<|30u>8`dWTsK1F7%W(|dz`1-x45)O$Ye}<(%D+pLd3k~umN)O1owrY7 ze(US&Svdfg6lArYvtHCI@dT7EIaR42IXE*$CZYN;H1lK^3z@Af_EJz2><+i41DTCE z3Tmzs*BUa!Ccq$jdy(%HSbt_5D`{x~8V73nV5jPI0}-$*Ww&REbK&1oH<-OmF*M(! zK-GBb3qEDEld|Ii|J9!+dGQTVAts*c1G4}OmD%A@U|IMG@5Kf0Z?tmqLH{-5hyRg7 z_3J^<6xHT-YbJAZ-JezWfgHCz7r%gk_Y5As0xJQK)Oyfv(Ek7^(L4{e0exTS7u%)G zg@g2*ozuqD7J|-k3u^Zl8k%CxIm*y|przus@O z2fB6?Z2i7ua6^%BkyjxmjuP9y08q5b%>kBCHEVh`zlc!K4YrvTp6VMB>T0VlqMC(m zo9Hbi*pfJr4b@@-9>h48h31gZ~^p4(7+ zVL^2t0otj_HF1IdS_S=6FsOD*9yoicQQB47$Zu+G*9pL}^v=zd>|v6k%Kcmyb7y<~ zCL}V2lVaJSK@_J;S3FOSMZuS?7J(HKxpv7L%s*w=XP)ln=sH;>-LtfX9?(&iFf`T% zC$()opMjjdnP7-^r6n<0MCN#@^7IymiZ24+R9CsaFP8maTuKs>y0dM4kc$Qe<~=gB z8;<+jS13L@4tlY-ftDneU8yS#b}i~#k%XUn@^Z|b`w{S^op%1X*@Q&x+kK5U0A@Pz z{TS(lbU?C??EMPqWW%Z!+!XGXg|t94k@;bmpEEqa(sISaCCTQ-=&JkD+yJ#-roWuw zjKL+!8A<~Rt=kH-t$Gb9e&Y(1Jq6;kO)_^9pNu^_1YbOKPMWPtWM&!AlFbMV0ib~) z79evB_6=gUXj5;KW1S-PXO3|i`ezjY132DlRq}BI*nglO^~A|a5Q{`f%MD2 z2-kX+unYbFBk;gX8BHth>xHJr`2jTKuWj$Zl+NA4JzK{Mz8;MZ{ov7HH2tH)`km(K z3Wz_h0@wJX)blpc^LTN``2^TVLHKVYFo!Tuk0K0O$T3fw8tq~M;0{e6Kz-N{A{1y8 zVFQIupcy6sJS$aw=<;<>)zkTGbZs8tUZ9Qj09?~rRH3iee|Op49)rmUO8%Ylhj9;?znQ*R~0xT~OJ-nEtK$d2Z!-z&$lV<1iS0Y=wGzs(Et zDot?$3IL(4VRm1GAALOvl#2k+^Oe=_1Vo$wjRn2~9F8;H?kr`UPB0}1xK#p-N!HZ7*;Iq%Pmcre zZ2IfJH>y}6P3Ib*m>i0`mr|R^z;ahC1E-g;aE$QC7^ci+S!glSHJ=%n{hPvUxP>rd zcL8We>oc$?m%fJgZDxH$wnY$ybg7(ASgNV4cGAB}su3&8*!E#S_iMJnl03wqL-_!Z z(2~b@DG&<>XrVjnuP4s6a*-T2KH|Ekb+=HUSIel-I7e=M7of(;(&Q1=<}E4LBw^`u z@J0Lk0Mu=D@r7?;P!}{0wpXv}8@&sll8jF}cIm$}^Rs}+ev(MC(d%4Kx8WvT=ji?q z>S#EE*RRj?iwTGBRGRDJjpSv55&O)Ye4d4d3f87TbNl5i^_34ri<~9bnbfmCYu|C9 zQv=5y2j7%)_;x&#-Pznn{zJcKgAMMueN_R3#_R+mN?BiA@6~0Z64hPvb)8S6akE{7 zoIcU5-PxqXJeeXrNjV|>9;P9CU79E)Xm&_9PLbL}3yb*CESbJ1pVl?4u_caOgua z3ynjp69sOSklumOup>UVIw8ES8Ax!5zyL`=Qz_hHs1B{W-jE~kEL&`dSt9GSn-@A5 z8|U3T*9p=gik5+wlU!Xa;Qy-}6Z^y^4%T%;(;Ir)3Tba%yP{)DYvzG)z60|G`Y;E) z_`pznQ`R1trm{I;|1VKTWFL;wNce%puKDsM(?@tKvrUhr5sj9A2Z~$we_4i|TQO^0 zh7Af6@;UyciMfvK-{r0oi&WEyMJB|Fz`rpil`mNq=W~{0aqmeLsoOB+j;<=}2Yq@e zix<9=)ZR8xNsf{KI^~7XE#Et8I@Y6DA5o*+ei9XgYhMS-jMycty@3#XoV-`KX%=hF zRC?Hp*aRv1*tBnCjkLd<)DyM=6 z0qC#@o#7-P9QoP7r1l*c{n?YF%V2nPl%1F#CdI&?An(_gliKWs&1r$ERYO1D999?5 zvr~ASY#>Sg8-(`d&=Yugma}HU=Lj=%!sMQRo1yE#m~gEh?0!BzA;!=3FAblWD;K8ql0!qH&~*=qx}cDauq%i9HJsEvLiQNFZ2dW|Eo6lDLy6V%$ z&Jc&zTfpR?cB-?$zC)Y!5wwd4IXw;rNKRxby0-g(-gg6(TLaA~x8q{l58>u+gZFiC za=TjARJ_v@Vv9viDbIX4(^du;YMix@+DVcKz`J8-;nD94${>;#_+5i7!%7a%y*b64 z<1-!o(Txy0ST0X^5R=h4FZM*EGP4Ro0ch>5gXYMv({d_U3*b0GM%*SjJ3-S`5gf zuzNom+aw!A3_y`D1l%bBxkx>*^9ochh4uS$p_UT5PkC#N^Bn-JJ zBXrWtgcMW%X;>~jz5_z_*jYf6K?*-!-q#?_emW4PPxT#&kQ2c)l<8&`+go(bOWheK zXTJ!rG<0pE1!r4&4?K9B>!EtNhD>aT)9IRDH2d4e{L?|@egoY!9-80@Ri-k&kxP*N zaQQffBJ#kt#r7W3r*#aVR>YwJE_$J_w)|Wlg9f)WD9))^fNDcf;p!4aWO50-O>5ZE zZ&V;ECQV?R!0s5jY+9oY4^B__fgN3L^K>*}b7 zFX=Ec!E}v6^7`K4c65bK4H4k?(Fuk~#f;k)kjj*R|EJWXkLEXux5ecnFeWT@P@U+f zmqtrhuvnvG1gq?laKWi zIAW}?n}6unCypUKOZ&MNJ2)UvGtze)@R1+i5jwutPbGz0{Mz%M(^wE08Ns=aZn$C5 z|MRm$vtoMpFkF`JEp8>xs0igKpfOc06+USWYm0ixQCuC zv{rifzz8dNne-mSQ31C}1>3I`Ui!@~y}Yb`xA=0+s#;#ZxXAIu+(~&YdX^&EwQEzY z#()2axq9YCv217L$?t z$8Y#mH19(Q7BL%@|Jz?#LFkZ&O1LL~$0bbn@n2tWs~GsRw@LpnoAW!{75)ljalGR) zXVzk%e)oFE--17T>DK-`*Rb|Uk3R&bZ&lo~_3yxK?XUdPPe8Q#=hA+ap+A?lT4Dca z+OP5fpizG`ZIvDWV;-5^!2e%;TK|BI+uvLOt3>1%qw>eUt@8eVf|6BH*q@+emG=C< z5R@#yM|*sp2$BPY7W&NOH}aaXZhpeLnWf*hetTxiy|d=Anipcv3cb5kr!nf$-UY}% zqmemU3ZyGp57iS++`UnM_4 z!>8Pr#7QvK@&4!E|J#4OK!L2lof!Uq%nCMv5)!%pGOXdhJ1h$T!@tUS9+%tTIj+PlSrNEjJ4Fx`aMwOg@=?u} z(wluxrK9b1X*$^xVYkEl?CNaSZkYQe_q*=28Blpe%zr<|++wPHzd`r#_~EMRiol5oR|2L+uQ7ei*27=bIjBKP z{qBOB-+}QEX6M*=SDj-UW3uLeD`xYysPM=dBk>*n>v=xe4k{nSUG-0vm*tP*fiZrh zrGCBIdepp?Yoa*7czJr0)1z5?Us%EK<}}u`#6fMX+Y7qgJDE@LMP#M%tAYJe%130( zcs8O6)3(IjuO$w(#*HLeHFqDtZQ)l`<>g#)fiDxeHVxLcZ5f(aS+4b_q^H>iuVo|u zCBF}90Ep0q6Mp-gR=uX#BX#y|@Tmd0L3D~5e_fbCklxFpr^7iL;RG!cL&I&{arV=$ zM=mD)RTrT?^~q?|dq{s-r-EUYZK0~RLbL4N) zvut&C4h9WfzFVivUuNUO^T&?L_@lzePh3z9L4$67s;DhHd4|r13QzHE+BAq*9-!W< z?))Rk1U_OkC}l#;D1bZpAVP25viYUL-LOj;6cihlp9I{|fp zKR58+#FTWk(cN( z2~WS~$p=^;#L~Y9c`pp#cO4peFy%E);PvV$lZh()UmgJP;92g!6Sr!aOAGhKEDpvB zU1{+1&9;Gor`C2w{9kf5`eeXfjqno}v#2rllo2=h$Ysk#Wr5vwwg!U&e%du^D|GTr zYWkt6BGp1wr_wyMFE@ErqQQKnjn9B&AN`OJyXwcmm2V6}T7ngxn6eX_sy&(#4&W_$ zwjblsF^F4odn6z6xnMX~UhoK3Wp*TQA*5p7VR<$xJ0hyWM(e0)ceI$}@4xi5I{9dm zqur{AzkfgL3Nfy*R)|4F$$R_f?kIL-b-BE+`;s2{CBOlQp(u$mmvEOb7EAS3_8NP& zh!{65Cml!I$oklO)n^W*PAzWos!?0$u^F6-i@YY5wDT7m^&gS@_lq8H0i`Ty2|H$> zk+92C7Nz;*Q26pDd`*WSf#3!esQxpQwg}hL-PG&jR9#e^PD6 zkc>z`3?ZzgTTWg-`}}f*`P58S-JmnQd#RXSi*2=ECE?6BbbaH7Y<1=PFVa`>I`ABr zL|{wAR~95KqEte}1_t;sTG1K(jw_fKVFi4N;j@vzcwmbZl4iH92^`^Ly{g zj3U7>_+nblRAZsl&@A@1b-&|Q&`q4T-Onrz$5n!oX+WasaEnKhy9Nf%(C>3ae}Cf0 zF{?e(xN;S1jSa~}y|5#+5wX(?%3gMthmfXU5gLW#s(ToNTO{kOB>6M1$^TyW_OoIn zZ)U$@K8m}Q=h(X$KT;M3&p222eoQk6fxSJMcBgMN$DdOVY(l;)yC~o1r-B9w6Q}fx6fzO=Io?G>)%nGKd2MVwWE)W044*jd@7!{CZ#3PH|mg`(l;h$npEtvKxNFU$h=2#$EkWpyo zmyq8nCoi)X0e3M}&Yv|}Ce^jqjmgcWS@1DCtyNmlvg=0@N&e%ivS*MjyYl3Z-h~YK z!fEB@)(`R?QsC~(v}nUlM1>}=lwG5^4}BPl#T`o@?1;)6>`guazOW1nmE9Gzf8Tn< z|8naRpx$kpw}*Kmi#1*JN!3UA(wFgXusgz`nxhM`XUZ1+k(pqe;LFKPu9P|3CeJY< zg}B(@I2PIBKQ@BTKr1@$jV>I?Pb+*a=~029-C7p1^i~iQzvfiPe7dWS;kXKTg5i=q ztH?!;8C@r1^`<6UHd##!>{m{@kdf=4Xe;|YvN;g>WW1y{Ha%0NafqtQQ&4I>j;AT3 zw0;y#nT;(q^S5DTEW40nnK?f$_iT#dtObC>%Tq!OH$#3Espq~9zm*5GyTA@_xPF9b zigG;*P(Mb1y_a$kec^jd2<&>H)8MYxIU8;D0#*|2GzCq2GNe2leNZFaB`4>L2pO)K zg-)eku=)5Z66*!$HfTLO{d$Ei8*n+~i z7Rn`^!zNVx4nMq*!1irdB<@t_NNd{{C#1b$Vl8P&*#?CsWGvbFc|^KtDK?p6wN=sM zEoGVM|9Kc#L}X-53;J)X(Eh(#1s82_JFuNcR!J+5=ww}AH+PhBz+vre;o&;Z?_)h$ zV*koNayaq+`?fCqxC%G((yozj80rZBqNh3=~7 z=tfdbl=0{(^hDpWvu1^`-!($6vx8?j;^MygT(~a|4A3e{!2rEvH+x_r#P++nW_kj=7~dO~2z#ljK>mknG=ur0QTk$F)4{ z+Ny8A%91lVIi>4e>{RPkc{0QeX3o)+K4)i%x@wuJ!_c#2rQF{d?3eq$m@9a##_^fGsia4ZKStB9 z!}&D^26p-4)_)r*Q`m0Wq1IL`qvPYyvw*R0YxVH-tbYB#d@6R^<-yp3Fsp&GsS11l zvFXJ;?^9m>jKz;rhLZe`=@*)+CojV8DeEi>wzw{q`O`5=-H4@;S_6Wcd4UQpnid+s zzC*2i)=2qUH#LEK%gxTg6#qkh%;&O7u@ZoXQ)Op10Knry$rq=Z$Kg2v!MB{XB>OaB zhc4ZP0!-V3TQHpdkegO-9QlZOMpjnpj%@m(A1W-W43KBKUbsqlfl+{8kbbq?o)}jP zpMSi~r*=VZ-Z^R$yjV>eu_xYt!gfXGWIzn6ZSM;XH?4vSQ~T-Hj(7}`zs|T;k(^<7}<;AHW#xkyFYB_R9>Kcz5uH#aG?d4Y;)9omg zHqEv;@>>gOyyg3OcdzP@2@CFuc^o6P>zY!)23A^pNS<{vjcubc?*F;!j~eZ?y%p^GAZvB#)$lof92Agy2$ z1D11KSBL-&1T3Ysc4*?z_(A7CLvt^F4Ryo>!`>3%Bs+I{R!q9X8tPS~42YAL=$ z;9PkV<&G(kH9khritjqx5(k@JuiW>_3H3XPb_ieYk#vl3R&^EL7;>S`Mq|Ere5FOj zkT%lQtm!dk>(GV7r*vsh$5p)*S9%P+&gyhL{8nabJ5s=G`IsGP@yLeeiKW%!N8F#qe^b+s$UXFO{Y4_@Ezh0%wc*~h^~y0%jY~`+*ZwEAKO#X79DwYB`o=e_0MeuIGG)&l-X` z=Br4AmU+y7URb_gI#|=OM7{G{bu@&S&kX2GD#}bR(imI`Um=Zq2a=~Q-e}7&+;uC* zUgyEslAB*oakpXl6+6j7P#YP7FD4e%*o3W{lD$ld*yK}JYEAcJKjvKKbafelD3w-S z?4{fsSVn!u_m#=vDp09$qHpcRH{3N%8|xb3Z+WTf z$G(FxT;wHpuR3RupUE09tJ3yj4bW&uH}g#7F)3pI;J=DW*mfcMUmtCBV$bBIZKu3h z=Tf2)rES$!kda_#iFA(a^fq^1PPN1;^afI#as`xBQqa!Km;<4zlmsJ2!J!gQdn zjq3>mzpv$YZW#J6-IuGzV__agClbAkAB0d--96o_AIgReA{a$pwG}IpgrN^Y<54X( z=W7u;Oa7ln{kx{WkJ)97WnyLK+vT)t;C@@MF*pDUHf?2goXlQW zdf$_4f>?&1-~O?>k<(Sf`#r|=3Jhq?{2u$hbz6+eS@iM2$=HagXujC9Cd2b?y`fuO z-MPUcF6_2E)aKE^sezun`N{Y&&K;sfNWJ9SwYGC<^K-bm`MRJ3HiL3%g{tF5eR1NW zOro~Rb(wLqBpcS~`CeoehvR2hEPcO6cn=qF+1w81JJs$UVRyG+uM2MeYR=^`g-zW~ zn)zxWA#%=UzMUZ<)S>hM!yWD`)qWlSx@CbPN&Dc?271nrVj`wcM!3MHNcof^x-f}^l%+4HUuKvki*z|2%Iy6f^Rani-4(+lO8T%+Y|lz$j(QJ>|Snv$@;IZ%eW zgzxZ8@x)*2u$a@}`VJP5;^yA(oNDh89inoMhw^yN6&)K}by7@fT^Vx>TxnXx4g3YB zUz6US3+v1Do+?=Rp{Am|;|fkXL*0%eN4pIOx}Bb%hriurQ#c#efgELJ zo{t|R$VaegemYq@7;^!9gVI9SeX;YiNdo(|@!eaONUjk9rE_+UpBVh5SWyr`qxe*guL#oxzi$QN~Bpxp1cEcDc#|pRA1ZUM8kW zDVm(s((96ORtPq66i39_OTJf$kf=3#JTTDMY@B^|xo>$iVrm5^4hol3R-q0FTSgia zj+MMAK(~{qBS}8~PLd-{TOS&0{QfJnXv4|+kMYMQoK1T8=67XnM;oictM+@P2e^Gc zUsB$b*AAhKWWR_J+F(xmwusQI@x9iH&jjz1vmBYU;2L9KF+PwVXFQviEoH}G@^U^w( zLwCE117#}{_UaMz3o_Afjj?Y;n@2ttK# z6Ar!wRurXpl2fdRr><}5Ymhf_HoBzaw@!!q(3NG8rqzRb`Wxo>1s~~&M%LU568Ng% zpS?N7v$ns?L+XUHN4UrHl?iwDl=NIX9}q0k+c3%^1yK>_J0=HSsLh=3F7%fdT&X`W z^~vx67<%LeaDXRpA4J-AzPQ-nisC}=JeJ3DvEJ}zA2uMZ z9VT?l_}#%kN>QLVG>;48i4)U(v@k)hs^#V7KXUCaRo>L$-o&Clm15D|$=>^YH})Ma zdw4ljgrjaSM5?&IR=!|CK&m?*`GeQ15vHjh8wu3<>kBXI3p2Z^UfdHypXobvxX(lh zsR;s#qHu_aW~2P?8l0d_3J|3vKN#~Uy@ALk7yFNI zW1iKr9w{|xNx6nI_h0_{LM^voPq2Wb7}+28!^_=WM$m6G?^~gx>2j^}@_8#_f9C1x zJ}}fe2>5ZL~3fJrOqjewMH@%Z7>m34#F z&GWTa@_gOJ&vrNNH0-;CH20qVTub}mP)5r;f9XJ}nSaB?pxJ!RZ}^4oRuZ9*p`>Ep zJafnpcUCy(3k^(#R~s@Y@Q8B1`Q1Ui3DwSv+9RhwkU{o*#Ln;G-0mz6p~EOy{S<}W z5+=KVxF2j#%IndINm2t(Y9zs_5XqAS|TCpIjTJgN-@7 zIx4gJNeX(|ippS*9d4xH$ab?tCC9jPj^9ngoN+7X!s4p^dZ~I-uQ&sRACBk}$u=?JM3 z)dwLE>47&G0OmVI+{F<{inj^%^VaEh`F|*T@35w}rhV9h1r!Ao5tXWlNC)YiV?#ke zL23vfsPx`Th*1#`0Trbem8OI$odDrbMM?yuNbjMB5+Ky@Td_P3@q16+@BQyw9Ix!X z*P6NKo_l7^sy>1}Ouf^Sy=_RkN+aYgKAGi1oU5yA%^iMjy)PJ9!Kd~0MHxYiX$MAQ zO|~DzotFQaern3_G|%P$(_>o68vT0Ds1ld4I|A~H_XJLQ%E+ovpDw_+%svyNbr7Pe z={k;$A03UI7tNII^Vtdz-Dz4q`%wO9r#&>;3$V*6={BW1QF=acl$djr4b{g&P3MA2o4%*dbN6P4jGM8SQZwz+l^u~EyuJ$E={uYq>Mj`i zGK)Lcef#O!ipl!DKCfQ&kRkg+{i&HnMN*0ztznWq6-#fj^TAEcNl782;XImkU>HXX zGt1yycP`iJ?03d$KyIJzr#(@_qqrD!%~P5UE0}4zrG{Qs3t3n&+3qlrm~Q85A|0CU z1GoEzFon9<^(*>6(*7(1YhLX%@GNtwb}TFN-E`xXYr_P8eKDa;GNs z+DR#pIE`!FB8^_kb)W0$`_|1N?T+K(Ne|P?(#Cr*eKWh3v1y49`)@I zwYy6Vc`}r$pCT3@H^Vj|os-CDya(U?3}<_bvI`zLAxG8auVV_p z1PN*9vCnJJMpX+mIz&wjYS?|zS@5|;919$@W!=6bzXgr1PP~9 znzM=e8^e<(e!xPy3Sfj@_r&KsCNd{fG!BmsQ>;u*Bm=o5&ezVK{9f`0T0dY}SEvWkfXXB5MGZU9p(YRJWIkqrA^eFb(!UDDVTf*u{zpwy% ziTI8EoVk9v`u=ODkPLeFD764INg!6yf5+Kqj`)#fefso08m-i(=m0t}5)vAQe;@~lWkNyV zw$d{j8|QV0?cDCrwN2OYzKAQzADLfRa8!h~6FRV^U~RvfSD#{lx zcY#hNm|;=Bz9zky&HQq!sBVU`QTiz*UO)L)mqx>TAbA?B$G0sGsEp=f!)mhjrt$Q! zk);lQh;&h;;=0SMe+iTglsrvUy&mAOu}>F1dKlN?hTo`1-x zL}-6ti6v^8HA|J}_?zju!((sax8o~rqT03&-CWr;%aK9|DpP91%1~XG|na)BRzj&kx19tPHaf1x*T|5Y#QF)3F;d?>gaY zK&|Ze$hd%!9$yS)t3A}9sl-+v%7i>8RRk_Y;{rodqBS9#)2ih%mc4wIE}+=XdpNEL ze~>{53|3tP*R(3pf9yxk!{&oYw=f+&nuq{incNYeHvF~o8`w(g3v?|Y-@j2%QqgUKGV@y&oDMzitB`2t=_yT` z%2eADF9#VXjo&HT?4J+K&$jQ_3KbDCj)|o>`k~`p+ZY>>~UA((7)_s160~b>e{A=91@o&qx&L7u2K|-`mJd z2%2FIVUu9%@Q7-`Z^oXK_q&s>E~WtbM&yAQr|;7INWY>u&2Wz5pwIiDIxkFUUX*h~PwbZoN6t+jN8h_z5Es)bJSGRE3vACvdSPU`WD)iw%c>T74Q1_mbcB{jFyc!rpq{u zTp9M11_*C>*>AmE&1~l#sk$|LpjVh>O|HOo1(=sH?Kb!Cw!fD{O#V-^oXTIh4x$}y*TQaV?UvJNC2^f zac;vPx1vV`&i+zDEBDhR!+G1?odAq8V?8=8;gt%(SN8ryMG2g>^MVA>7B-r7wtaJ;x72*9?D!W2f(hMsl!t>am{1x< z8coR511jQ%YUs9pwh-izA;0xm&-t8vWWIPv!WcJqZHbIV@N~^T;Ak>5%N&joa z)o->tcXFyEN5`YcVUV4gjy2G^xZq}Fn6{%VOb}X{fAo@&l3uBOze~9rOr@+4E$7a5 za)(xi?e@MJh}FnQj+lRZ@E)i2?J-*2K`&W{qBl0dqlkZ6*G4>6Yr7X3ipNW7@s=Jz z(U!t2X@n_iE3tRhbU)XLb6Z9P_Wifg!OZods%Flw&P^15`ajcC9f?$ojmP45PQD=r zSJRe5^2?*`i-t!kb zJq1W6cHGQNFzJ%XT;NkT{u-Vh*DfwD({eCm%PK2{0c=T>b+c}>uJ|0G`Th4S0NN7* zQR13eu|crXdS{W6{y3|Z5!TZhXU3;p-sojCxNOz*cpbs6^77v9p!Y zoUQqmtDlOO;8+`R^I1&p?jz4@9g}Mk@bBULb0(PTwsZ!_EHUcZe1@c_K|Km ztp1WN{=H2bS0R!gD0Jm`xluuu$odVI6Y=;R;nW>EU##3u!6uQH?O{>4Xy&ZCQO8B|3!KblHX}5k$$(9F` z2OU!2_uD05K+K-%?Fha3*T{0P#_zahDoZt$$c*_b3(T=Zzs@85W@XdCvNEhPtF&V7 z**OE`ku;tAGmla_AeMT1#%~G@@L`HBYJGX@F_Qw1a^~oZ>L#EH5nTd z>{nagC@(L6NmN?!Cl>LAlz@4e_~G-0hL8PsgYNOY|@Tz-O7)1 z+G9KToPVR4CD#kTQ;|DnS}64Ajt3te{Gyu5YC~fU;hB4!HuTcBGTvVu{Z+Y$daAs^ z7P)6pd`R{3IjGu6F{dbai43Dl^y8o5zj$Ncmchuh4>loc4?>M+ftr3cFwI0k(oMPe zG`oG|dS>T%{q?z$?Cxx1QKg7w_e3%IxT=$+iTtNY69JPuKu4JF>h`xZMJAxe1|9EI zNZh%W0Y$mDeGv?x=LiLYTG52lXF=*IjM5u<`Ww)GAeO1*z5!|RTNw;;@}h2D zE8X7=nNJ8&U%`JaEtRimBb0;<9{1=sdZL@wec-@>kzkW8Gmu#;S!Pnj>6)(Bnrzj2 z4sH&3{JkUZdg%LrYrFbO-Pq<{WOl`AUdLX0A27|?9Fw4si31a26-YDxWYMg2ZAuo= z3CQg1Y&SV@OQ+qwM97hp$Dtz%MrcO|fvE(VCAwx*mOi3*%|euaeli0uu33;OPA~`< z?-^NZ8nOhmVYs_sXW1a-G})K@QLQo(0)Z^DG1ifT44Fl2sM?KT#jJGA9#T3Vq*o;B zGXm!pE#O-|i-aBGVDR92{9q927F$ti=_m`U*?^}{M@Tn$jQR|28Yjc6MD6A1undol*7H>5d&Q9S`~+|XfwX&uesoE9TY7a- zpR-zqe(u>wBgsuy6QBv(_G=z8O0z)vWMLEGiU~W5;Z936YhqzcNILqJ3|?JMp+_AZ z1U+wy%m$=HW@yY~XFL2_uafM^?6L%1C5m0A#q#@0g)zO(e_TUoI%WMiS*(KUoM zxTUq>uR+{D+pVJBscu~HNontWEKOWPZ$7!KBI^_U#xB^@`lyzj?bX11^JUlPtS}+E z!xbmY&CQ+0KEJ&7&=Xv0>eTPnRWRWLu1*;R11E!tM4~Rl&`QdVOo1FU<2L}f(S6}z}D~%+4SI{=Rx@J~Ly3qb8r$(20>QqU^Mn|?}!i>Vam*z~# zD~Na2Qu>#W1xLctNUit|yPFy4A6~`v!POYhyF%n*Cy^}e0S`=>TvshsM9wmKx+JK2CqC)s-Fq1tClAd{-f3xoW%s2h# zai=(9_htieFuV@X00}w{MPFTz7x$hoD3H7Ht_K6IZ+N-tMT&;*c>b}t^Xq+BMP`6mGTEli zsNhtrH}K*v&#dF28esh#U^9@#IR`ju&P8x?E*LS8C@_GjnXXOI>+wu02ZaLMFxMNk zG!H2vQLDZT5ntQ2=H@bdOBCC{4KcGEI8yRa;$ANc&uD{QXx)ZHd9JS$<_emNY>EDm zup<%5E>rQ1LmGWlf28625iVJZMj0o2Wn>wEOi|mRU0Jjxag>t{UEper;}EPn!feu9 zI)Q8FsNluv!*NU=D%Y-^{(E4tBv$XjgzDd;_-)m{ho)CUJ<7OnV8(bcxH|#@wW@F} z&v$m{NyE##1B$R1jqyD%=`g_B-@gLyk}mLY*VsCFX3!+YCOw<=j(z{VxCNAEy$i;MJAWHRMJGVY`@+Mo6&jR)POdrZ1Zs;Va3rVS;` zP|wfjk~Trj-p_)uR~m@<=Z{WDveiH|UVTd}dKB5%UqVm5W{yXxuz4GfU;IN|S86Tm z%c2776Lzkupib23t1QB^wIAk3*{nQU&&6%njC@mDmUHGDNX@{|r1Iiv#>G4{*+2sz5fQibbs*dOra*{=|52g(O^ zLywLy@*gy*R1a}WzVR&kf#tFb2by=lz42MGO}oYEzdGN&d-wV8LF>R(FxS`9ch;N( z^$@8pGL}o7C&;J{`AL1t;@0kT}<^>Oe3*fLZLo> zOEKMh_S%)VdARp7v)>Ot3Qge-gxEFzHGa#| zD>RNpeWf`D^rY2+uH$Dpw&J>YD7dA9%duA|F19LXj;eB8O`RZ3{aE{<7Z~&#G6W6= zxD#};8tOkv?nej*2rB7$r{Ftm>mbaWhXQp@eJ^+M9-MQjRpIMMY-5F)30^1R}g@MEK; zGbwQkn^M`&O}uwjoT}6P8M)WlCL*T2th4=ewbY@|^h_&FB!g7d?8m`B`vZ0hi9;8` zk;aO{#*ZN6ULDQkIr;H%uFc?7kBG&LYR6j(iE}ZBW_p}`a5gxl^#KXeq12F$07#dk zqrt&shDA&nzQ?oiDR*l}w27%xa7uOE5j{-1MM_TuP=`YPd9(UH71aC7Ha3E!o!_6b z#JEvfxqSoYK&X;BIB=g#-7{U?#}1#e1bwp)wy1c=4CrW!XkigHOPL)7DHR}5eYU+R zWzw$S?{1KLD0gn@#H4^$P2xu7O0HM?t00!<`HMw&Fxox^g7JF2({)la70aJlNN2&E z{VAPtMM&v@(;w>N9!_NN7Ig$c)V?e#EpDjf2A#|kx?|K8@!;lx_R(P^6ff5-I90ZxF=L%>H@^60)M01R6!Z%P^;!Q;^2jXX zmmv?SgR6A>+9#n=WYXk=%R?d6X`?ljj@#clax5e`V^q{T5;kJCk$O#G+~xgM)!*&t z=Gysc98mWSGW0JxjAS9_%Qw7kyy=K(!`?M}KXJRHP@4S|TdbP~GFWV)8b7|I^ux26#wv}2u zGjV_XWhgdK+0cKjouljwuHydGl|5y+ay=y}()$c&s0epSh82}NT}<#Dw`7$Ob=567 zKxPJitWBgUuobNvs9zUD{7M9yN-M8sD;gaMo{-Z3+fky4q$e1_q-mQQU=|iCzH2xg zC;XA%^Ef3mwoi-hFKnT;y0xaA8|j~N@K5**wU#f=uAZH=r)(XdV+ws^LFhkX&yR_DKT5|7gg%1s#y##XO14H(f(Sbw&AQz1 z%PZ~JJr=mk1DCXYe$-Q%x7Vhtp{*}_@nx$n8l2_Tny{e_&{^M51GYf818-yi+aaZ$ zdXR6v>pngs^}V6v#uR(lBkzC>c;$>bNd~x0Pb)8LvOR}xY9to4*^C;xAzDh6EuoMY z&H`-3L`Ck#LhAjHDOl_#)r=mYuE)j)bRACBpXEuyt@Ds{ao{}<;T>BzK-VL(v!Sx|<61T2`%c9N#c|f7@pXNKa=N!5s zdR<^HXi%3Ci0ZOh86I=-w*}!-)82s5!+tW>^Zhh7)jNS7k4^G5w&N$f32i@A6ISjh z9lBg(AS2h+LkU>|g<;;S{xgr_-2VOwTF@ZY7+CXmuLJai zO$0`1)6j4tU)h{#Lao|FX|kB}LdF58X>K;aQL)@YI&l!oGyfr3%ZPAt=(Uzo7hMmHYmKab0{h zQ(ldU*;`vF7?mVm5|CR5#}M1A7suyuL0G(nbk}!^>u?pUgF%gi1lc+?Ra6$1^Ne;k zyu+hzLD%xK1e+szTufZhek)b_g~dEt-QV2D59aHhNzmJHy3voyDjdh$IL`}rcZLi^ z+Qhmk4Y$6M`I@r|9Qk!>c-H!G`hMhSX%w#3f^x9bQZdo z6FTs&7&mY_j__s`P0X7Ciep6bp$$j)0kh z>?qJjwb*NKaDqRgDR%2-hlt~fJluHc!;e?c3Ek8@QqPalH1wB7Q7Qs8k+^WiQVe-1 z<$)S!rsCF;4Ud^+@pY?I5eu0NgCicD{Z~*i-;CP{C7QN@C`mzN?2TTgb1!ty5SOVP z0#ObIvN8h3MBfxF=yEOue$s5g`$i+(E_CymP9TeT#cwT>EJ*weB~2CJEoNCzBbAVN}|JlTDbyoON=pQ@5YpZ^N{#aL4ueIKzX1 zG=81S@gFn$5?e4HHlcYN3+s=dGH&x_FfZFOKX7E=?ntNZHmeRs`|igwQ6bU2xSQ(5Hpe~bYf7tA#^ws0L=T|{p9 z`O_v7wm;Gval|?pUtGU|Onf};wStcXyw9u7*$3avS97YKf=if6f;K`6?XEs@^Bkq( zRW3v7!;0OJPi3@k%OporcNC2GfY^oJ?!2G-V0}xzjmURQAMo|g%;^v>g%pm&F^r1) zhlI4%X{-oXfh8~x<$r&A;XYOLU$L2LQITX!hNg*h5C&a1f9Lu5*9V7;CPrO@mYigT z1H{_BdZtM{xN-CSHt7JFTi0sppD{6_#6T3Az4F$a@puu2XxanUNjxNgkm3X-|*Q?0_wN6Cdq zZO+U~Tg0NQ=QFwjQF}fohWBYHXwJ27H;LC9_)CvciVaOqfdklnhzTeA{0CRx?tHs- zm@Y)42})na{xHQh|mcA6c#wcXbt;sn+Ci9oB6$l923{LF(k zccw#tIl#! zb(%UfL1ss7Y|q4c*vV|$Pzc)lE@-r8N8WQL!s>6^^%XcwS_C!T*YOW{C2c96vXQ3`hqH-aAG~ix++({1%b*ex@}XoJ;EH zh_J38%no-KV((F21IJdd2F&V~RDVXsfCM0Ab4d3SkEOi5pY*{y;qT{z52uQ%g6eC+0O;%TA(oOdO{S8>az z&{2)U?m?WokB~HvP}d(!_6OtZ%!vH z_$zFWpNeCQ;>qblF@$NhJk_qHt8wD7rzUq~vMdP^aE24HcK1Teud>#!{~2-9)YVt= zCgyl_wZSKc(MOr*j$=iwQgRhmB(OxJQva!n;_N|Ob;ET7-diQKPBIr2^u`Of)LI{EwgbD;1M}@o?V=^@td`5k_bYp|_Xv9B12KqJ(d2?@#kBHo->+4}w*JQ5USEk0g9rT%--f4q^}1xIH~aU+)agx-$@XFoa*q~M=4cN=s>^k|#pZD}e= zGbG3rdkwti+c&)r*W+Kj?6fmVpk5L)cgYu0e~@gg$dri6y)@T3{r1RTCfjsVt9_wmKD==i z(td<7uZt7?M?H90)1ssc&9=JG`P{QTo$>Eq@57NEzSYx+s>;)p^vDt7*{+LT^0q~K z(wXDmtGL8L1DW3(`(?6Z5B(LSa{_@GUw!CGmg`h5(Uz?Q@DKOB2`ZtDWE5${<`(fC z6$`u37zTDz9M@63_wHb5#qpGLju@&CXkd|2rBKb{v&Coug-HB~mp!qVY-k{M>cju^ zz+dmu7XY5SsVw${j4VXx{&L;#uXvq4;hcHT6lbBc%4O_b;1-i}(=<-Z2God6ra%lP zRIvXY+GD%#__|AU7gFfB`Sb{1&)g*={|^*qY`oP0=9H-#L?(YA?jK!qKyL}KZn}=) zY<}2M=m%>x@S%4qJT0|PcDK0^3i26>RS*tJC_(vLWPZ4tcT1Z*a2~8(ZbFexh_KnU zI*Y1Bv8216_hc!fM!o5NyeO<;dJDnqCT)$#VEIVNmCD>ZP?}#G4W+4SHU58r7Y9hR}8mea>SpY zVWf6cmCTgMVDm33LE7t__R*@GWF{$4@Y{X6cy0^jKlFFE&zzJK>1HWh_IO&)w{fEM zxgwPg)uPNsumkR}iJw_}JWe+uQ~o>v4{`I7a%qmlm?I8ssAG!3^Uuin;#KW9^%Kjk(RmJM-m@!)sG4yq73mZN?33 zLQwP|`Dw**hLh~kbd1_k5VcQbiTfoOMSQ@tgc}L&92Fc_%%}H_;Fpsk-Je<{8ksVW zI3IHa>ONm6*aN=8!04EgQ-1EtM$)Sd*Pq=Y>q;cEbs7f!I{?VvU!fvf9#fhBx_!#@ zG2X_PZlby6@8@SsdXBl<%?M$J;wmp6)nA1M2J7tL##RoeeN zy02@<{IT*CZP|9KqqLH|j7R*EQ06$k8}gEYbtx*pr#oRob<|ivo?&%vQth@X6}|df zXXX*#)#aNJy=oYX60cUgSzo4E1$?c!WC5k!b>av!1&C4RJanCNm3-2HGKA*&^Oh$c zr(!z}R*f_1a+{EXWbCLU!Q<&bRwx6b0^7cbOG8=Ez|pHqSix*8#%&g3-e%S~*X+b) zE(W&sHj06Vi<``X$v^KAU?R+8t>k;(c*b$8Uu}4~b`d44qw)HAxmS6XU_siN*80dh z^}D8Ode0ph^p7FFKm+mnhRYJN&#iXVe#_q(jdefMa;k$2pUERg`u=kadMHb>o@vDW z zI&`-Uo*JRcrK3{gbu>XSq+KE8=jda4zrfbLj&3#4#r_zUlzyr*ROty{OztF7=L-$a!?>zSq;U2)zie- zScd^m>5Gh~54#xyJ;|?S_t#z++I?#s9Oe zr2nXNKJ+{k^At2JubP*6CfqS7cne6ZaVud(*zO>z&`qy7IRtEG4Kxsq-x|H!HLBo| zb=yxdPX&GS4leA71ZEkU+w8>rl&AW0_syhU2TJRmz7MSTN6UYURQGU(<5#{cdCfL6 z@|i=}8GA>L_G+70^&x)*3s?vP`l!ATqL_Bk#Mebc-osH_zbI>6ITb4=_Xm zBo<$n{E4+!o+B|1aL&-D!!JKlUJ?;afL8K!Ok{yYUjq?-Icd0O($(#eqt}`7a}Ca$ zX4qyY7H_#r9@tWba2epo5YF>-a6;|e@YjSR9g$hy74rlGps1lt=t;3EyCUI_nOD2_ z1t9^f+_PDUYR~TN$+<|SGEjb85iR0pQ8bkQT;Azoc71U=2$&nH<9g^~?!)$T8hW>m z>ANGcmC7Z2oiyS4gRQLf4a9WzdG?dX#d&=(W<{qukIVcb!EdW^gM}QCrW10 zuwt3#v(r}Z)PV_)F{^N*D|BsE724?(qfg-ta!vsM%dXYzSJ+L|O#SyO%1E7PF^Q%T z%ey);T0SAwSzEvQtl~&H+dGsCH1J>?9Th*XqiIg|^?UP2ptbmJ=_QL#Z@k&V?LUBg zBQeTPAP%(}1q}>;`Nt&M-G)C@2)@4exCZ&xm+Ui!kJ?H*xi8}y--L6T1K+okKxImT zeB-p3mAv9N4F*_X-qQ6BD*SuZf?fJHQrNCiP z^em0QV-i*9T1_A^^znpO{WDqdgnI9Rh`I=n?;cz_TUC%2goXoWToh)^hk`{otuNc020R)mN7e2d2MU{HN^_s%@ z+672Nsj_7tCj~85b5ETb;m)|WD%M*Sj*`41jEvhCcNRyGu@Bwc{E0l8c@)>Y7(7yf zxThFK_Q=Q|EV-Sze%>f6?p!pF%EXFvIxE~FqU7hQ?`%HaI~>g;`GzZ=(UJwR_c3fZ zC?57M4Y{r-HuYVdziDRblrRA+I$q&Az$zsv%Ayg+29yBj+Y~3KeHKutW!ERTv!d;|>ddMjh&^J>mbflOT^*Tdd%gsV@ioRUpWK`=v+jYf zPLwPVZ-AKmdYWv1$>6u(0)SsZziAxV6|nrAA|QcJO?4Q6jmt-`;Q|?^&CGnR4t7-( zO91Ho7;}RKQFt^E#W2q@c?23Sh`SW0G$Vn5j~)QT9RG4L+HvbSG_ZH#uN;Qmi@O_i z9#$R#w9)u)-^ksgxhf5h{-zx9y@K;E41wj1*VK!j4dX3Jt7t3MqjPXL_aIe-8{vK& zJRVFVk01X2nDqRF6<0A}c5gP9@Ml*e(Fz=jYxmBrP?C5$KadPy;~$jW?spVOtNgX> z9S#4iUz$F47GGR8z)I+6W~bluSbiE6H^uoCN9Dvk(jP{6(z4P_?VgJ*Ojjz@DX5Rt zoNHd1{_MK8hGzk}VtT(-2Dws9j<5#-<_n+6`bb7QO*KsK*`B|tK3EpZ#VwVZP{!Vi z?9uX>wuu0^Q%yk2oFtKUF}{QO18?rY`NG}rkz@2D0&AGCCZdcI#)|b7Lcf@|Rhg4sB)hzf z-}kO2x<4TNrLzu=xciv_W00YFgGzS;V=$gr>Me=`(t;z*wN&U5IjmN}q! zuV{BC7vc?I)zu_c{mGAK*#CrHAp+vSi^Oo-S+_mcr0hgan|Up3Sf^dorP~k^h$i zw4g0z-ifZIDiRxbpN-U=7z4EsJ*ZB*Ytrlc!=2 zYPU;aO=LBfnrC=R;})WN%ue-=W{YQCK(8)$iWFz^tjf}47$S=IE><80zUrK-mI_^d zF6^q3S#u{uxURNqQVNY`4K4HFJsfsZ=c*MM>n7hC&nv(wmELn?^zIf6DphG&y$D*; zFehL8Q{(rBNW^?5Q%F99|BIe+-@x@UTelC=RC5;?JkAhX)#R%4sF}O{r8Q4#%>P~b z4sg-eU}UXJU-SE|=K1U=j287;VH9Gr=@Fox81Loy1lgUzRI1W!4Y$8^u;Tdx*c-J6Q&P9v3K~RYMZAdvCU3CO7kZLvCxrMqBawt)6yB z^-y7gblcyi9kex9*3iVCTxh;e%^Rn!=1pWy$BOFYHb=DveRteyvOF1jaOc-CG_q&^ zdM$(mko-N>eVgpb7N}9!k)K;3^`g)_Ut=8yw`i+`_rBw`b9V=o9gtkLM=gC2AervX zg9bJtH85~%WI8awT;9|RKVx9S46o{Lo7(+#U^Ny3fn4TSqhwmNyZZ(hp67PRJ0QB} zdj|j*%Kw`6k?FVPGkDm!F?ikqGEtein#bfk6poEJ;Zuhvf~>pEmPC9$cY^>JPGOsK z4uA|W8$thFa5ph9E!nTA2;(v;7ggB#G@e}B9o}UQ={YKq5E%wUn_Ua;oa{ms_T)GC zIAV?4@QH?PJ};)d;Op@>L_Z4DN3;xr>;>?5fvc2g5c^%AkA*D+%yjNHj2ta0sb~E+ z(*Ogwu@OSc%gra*`KAj3wFT5ayR4W5KXz`((p1Q6raPe(AOR6xc6~#fPK^_f8+g3L z_GYtjE;wGQ9(pq2a991Onw{FZX$r+e{Wnx8iI+0JtU$!c_9j32HyJ~EoHVw~bZZC$ z4A9}SgUTe%h?yNZ37NdOL6dO+v1YXMlLK|TV%IU@Pw^J$bfvHxQYQ8ep)4Q)PJ3Ec zweb1QGYSlpM*cmiwAD)AX}7xx#585!0MSu8PT2ioyvOTsCUX^#26-)++Xk!HYQn6H zZYvg~DcMfV+df$fsZ0U@z)5VuhXp{x2zLgJy@k_#uxL52o6X(mot0ajgtZlULkg|U zLq{23C~pPA-jZKh!Y}xnjgj`^OE?qW8yT!I`fRBn(nI1^AcT!Cw9=Z9M2D{C(k^x- z4je0e)?dh?qWWG_s$Eqf?Bi+Dn0@y%iZbHX%p>a!SuA>jRKyQE1#ReMMgj|Di>Bj9thmcOQd_pXJGUlBAKhH=3OZZE24s=CNBtKU zi$iC;-VO%_B}1T`-7oXY`2G|1RZBO2p}tjLEQirS zo|e!Qc1p32>pZA*9^F*#C+i2*RUP^aqW4trs7s_U_gSIR1pi|!HXFuXPCngIUs03a zsA%N*NK|yK$-_-|_EK5*=7}HNU--COkO@bu-<`ES3I$eW@KEW!x5+ilO~$o3mFi*N zm#+G#cB{BTzB9$=5xFlHqJU&9x$F*7kD!aVFVmbkx&ZxtWc^sfet&7Wl9_^onkNbR z*1p+FONgI%s*r^=L7hEo(4U( zzrs$3Z4Zrn8)@wkzbaKQ-Eyw`h%zN0@g4`blMwd2`N^n-%G{;6=l>pb|NSL$G4@T| zry{}`AO<&~kDP3?Cwss49xavY@{hVc#R>GHitVOCM@Pcb5(sASg^`0%6eJb~srESG91jSW&XSYR{Q}UH!kPkT<~c zTW{oNkWo%M{;y3&0z4wX-%A0^5nchAnP1qRe)E7ggpD_}l8cAIW03fd)PZU-P1_h$ z(XwB%Tr=ZSChFB)M%P?CX);35n_b{l;+f3vu#t zCoVy?%=(w6N6dY$n!taC8#Ahu$CZXBq(!=!-5ur+#?SB@Kb5~L9q?@1O;lnIzBBV~ z0k=4_e)HdxZSc+d;G6k~3j$<8V2*R470Wd|J-NJao9+e9!;XtW%KcjK&-)+8bFmI3r7CX*S3F%n`!@?VEH`@ z02(NlElb02!8m*b3hK3N8$MkaCxKOafliZv9##*xd$PX18&9EuGjaL5e}9*=16rXQ zO@jDUgn`g*j?r^tsEFf80IePmR+v}wPScF2?3}GEe9XvaS`7gGM}_UXYH&UFObt_Y zTHnoRyWt_+5tjo~uH-t?X*u78=T*agtl_cfLgmRGzjj0(_eY)nC;vqo)!3r%R ze<~|X9otIL_HL1EZ|kc7nbI@I={Oc2JPZnEJ|?O%4Y~5Ng;5Q4sEMryXazYCSk z4bBQirH#&(g}!hQ-{my7ng^rPfwsB_)avkyAHO?VTMB@*DurTh$HxS6J5?p=Ml!KS zm3C_t)X#>e$F`COo!jmjnUcuWtN)TKoNt-j@`|hYrLAiZD}6~~+ufg#Zyciq_fFK( z{=Xr<4j)J^-uO92>~5_=yg705RQwTAhAGU&eLA(XETM;)za8W@Z51JRd+V}luTApg zoZmkpRI?XSdKo^yQL>r+nJa->yMv=Qy=S!Y+kk*d-Gi#dJm7r^0qNvz>$?F^9|jnv z-k`;UTC%@7A-V=n9^wf;S5^1&6%1maqk65$+}uZfkRNzo`QGw%8aJwaF;s(o4+;o~V3 z2=mZz_dUgo$TG%_B7(jy#6=7_#%d zs5CqtcyLu|t&quZb`{Ja>0q z*+Hg9<}WIyA1xZVIfZvFu-eA_ZqzAku&A38(;OO5Y$b17X`>*ymvXz4sXyLDH=YDU zgqH>)yI)LKgA;T%t@|%J@}EDhJH{(i{OQ@bnrprw@XyT8(e%g9TgOM>!4_IAq_mz;U|yz72LT#i<%mcjOFdKe6UW24s%mY* zJ%p3uj86LCH~P8KA*7^pnv~^tCQM6b?xX4B{@U&D`4)86UDYB&gEqWwW?=26_(;re zoj-gyaX^s+NZ#rcsh7@Kk|X-wir5{a5$->lGQdt^BeY)L^7IKAujmaCj7tVJhhG~& z`}he1C&MmRlzH*`1)4#VUsPu_*lE!Xh9~G%k($)_xwLg|_fXh`9?+ZN6zeAvv3txn zs3%A-wbcG7cz?o4OyvkE7V0hAP0b%Jhq8q{`RwkJ!%j6PurOy)=>Hfc)g&m&Jr>Cv zhjsuNp4|7S_wt1@xCgiE&Y7VYB45V;dnoEL0T17VU(+SiJqsD6SHFjd7!@>H%6t^b z5(mvO@;5{aYq@~xND2p@)-L8e9oL=@(lA&WkSJJPOFuL|1-*1*UN?@r+`&yOVz5Cz z^}%e)O!#o!+D#C&$RptH<@j0$QttBwi+qCQ{ZzU}rju<~PR2*YLsVp5=J>Mv02aot z2|{59bCJ|5PExW7hF~JAG_arU z_&f%OAQk#1xWAZ;lLAr1ZF^1Z)pfiSCIFP1DRX{FX7I!bju+#*bNsh#pwtO?AM0u4 zz{mWu%5>k_*Z1^8gR!KR(t0_A#lMQ!{j8dfQ5vzv(VN5FEXn#3#mD5 zl)k==%p8QMxRUbjT5n9mvpt4D^<`RfL|8_mq|fv5oXQDHvSAN*J?2>r&T_D72aZFTq=oDuHdLq3D03Hao>&F}iBc7O7J zp$;7?ZzYnA3Cqe(0kLM8_=EOs=qrF=SESs;OeA(ulhVz&Sgvv(t2*aPPPRW3X7gCl zQSH3pv65C1)}2KJg`UW_f+V9GuQ$8>htWMqy_?`z{B`Mss(8kf&kB&V=auI#l}3qlE5{zZv-Rymv#zDg z?x%9*oTVbQz-!Qj^p~?(c}eIT{D(Ivhm{!vihKMaKU4-(r+>Yw-TEy+JlSt@R+uPl zk@){V>Mo3es}+KsCPi@*#s7i2E*YzT*LE{<0NG(8?lB@e)-uQUGvf`JT5oI&KcszW zVe3P!jg^t;U{}Jx=>tWp0*cr6{Y6E84ir=d%FB6NvxJSymg!PYG5Zvm!`ZbZ3dQGJ z&8vskzEke=vg9J!`tn`-P`VUo4D1CA1-I?(xxQqy;a}G=V$osJbxb9vl;Y7 z5$_rq@4A{vio4TkAg|v*xcyKWCV0P~{Xn9dk1AoAHX8*R?LsA$Q_(z!pN)klZYZ^+ zMC%f~zbSY{_~I{o8^yt2+>+9uR9OB*s-T^Cz`XlP=*sD1#cy8u&fov;7W6c?JT=?A z-~7yZaiA;cEB?$ZfAkM|t}l5gz&S{QQvR~Dk5%9QFFV^yva^LV`eeEa&=yvto>qN) zm%DxgJzi5;+zpEPss|tEdT$2KeG~?3A0OrsP>~SYdl&Rx>9ZrE^3gxbNlG67EF0Gu zdh5+ZX!#}LqhfM@bQMlg=5#rvmRLQ8!Ia&F{I8)@7D6Ebtw93cg*6X1&14Q-C$(?& z6p7mQc6Ur)2k9Tc&@i%6M!3%76WrgC*@52_nnNUiUD6zfR3Bx&mqe2>uX zVGtxj?cw$QI%fjZm#1y1idpqM3Fa1*yXC$@5^6y<)p+iQuAx^3hsHC(C0_F!*XzI+ z=Hr}wWQD&ZqP-N$JVv}J)@?F!lxsDEbUm+XzD8fBpEKmKY%j%FDK{RT716m+Es9q{3w{um_C z1NqO^2RAWN?*ce&pN;AXVoPm(An^2t9~j@?ODfrpq}-ghDD5$uhk;J$^ty-%1&~X| zNsGl$tex)o}w<*j>FT2!Xv-ps0JGf3+7oE)m^T|Wz7TKn~f zKPfRDE@Ihf+wXlTS>;*7{})I_XE3fW}W?gUmWAp3Zvy zzX(Z(NU7@?&&b_H3uV+nRxhlXw2RZ(C(mloy*7gXkF)O#YclKBb{HH*L=aSzs-mDG zARxU8DoRz6UIh`65_;&7S7|CRigW@NigXa^B}kR7^crgDB=k=5?I*}gnDfe<^ZoPU zn#q3l-fORQuY0wXK0rjD+j(HvfD?aEoRKz6B*Is~Boi&_Q4B>Tv_4l}Z*=3!`7t@* zKKxtA;5hB4)rs$iQY-$^Zsx}R0C!PZ?*-&F)gc3IEq#@lm%$)5i0$pBTX$*a5tNYv zl75;7PDU8ab@U!PVii%SEx(*zxNNlb^3}`2C{`kBb5QOkeYv}7gE}ZfBmAA7H)*hB zhWm=W^VwI*_1=F%%#pPo-vf*EiJe>44%a&jtqX-`r_pFS(GGeZOFVuIXM<+#$SvLQ z*l>M!6+3>Qe6S+?he;lP=pGR26NpUea(vvyTbnBms9*e=KHt7@z3k+DA$H)WTIg06 zRLk@(wO)HF?Rv0c#fF%o)6;vZXwxHp#p!)qNdF3y9pjZ8H0Y8^m$?UKGQ>au@~C1? zQa}2g<+iKFzz~Ttw-)iclJ@27(Tq68Tf;Bb93|1r>jh%k4D7!9JTY`m6A)V)J=!=h zfm8}@LQxvIqEAdY>$I(&1-?SgYRM@ojhpd%6D8pQw(;=i)sfopcg*`G*WJz??hlJb zeG&Jw_P~|Z{7232d8HdssrGdSH^nN0>^92j{vY(qREyqejW1A_lGeGu^*|xAmHSq( zs!GI{Ssqpcr9E?UOip}SP5jQ6UWy@ochZKH`)>leak0K(oxC*Zct{Xg-BttnS6fw4 z1f*DX&Rqm1?J*DV&|?WiBZ>z`)JtuP?WtQ$YQ>1L5tJU~&R?`O*fu+{W;ABE7istz z;_V%6=}9wDq+NHO+XoN1!MxrBw~i^odx&baPs#YhhiAagqa%h%=qd2Xsh6Q&CRq=T z;H+5aHWtF{BLPt)#pg5(4=u0$x2z-+sBG_5C--iS=9uvE#4S+16M_0-``KW+hHn3u zTn8~gn6SL))3AkH%5(wd6l5j$N@e78F0Kgnx43SxJZAqx1ATp+`+|e=aS$pWN`=A} zX`$LkQWESRgvn*7iVbtek!lx=$}Ihk@dMd3Vp6yB&UV4jTqAy7iTpVl5QJPt61}yH z_ijIxW;|512x4@PRr-5tlS&TEuM|R9Y6Fz`Hz5c$P-TyIdGIDHZ!GU#B*}RwE$Unf zJE_JiojjVS7JM$RbAGC-VmYEDz#-5puRIYtd15};G&7WzDS$9}i|QxpSSu(KYwFQK z)TSj!5dcY{9PGaiM}rcjCmsx6m_m2&n?}=&$W()5icuUx6vrAskn~6pC2FiLlr@0M zL76zGGi1nkYY0C5-KV0Z@+KM&H5a-inBcpNwiZ6(S|f(j0zgTKN`D6B9P=sDU0w;P zw(~p>Xquvt3hrHhTs|&rXC@AmUA=*!bLe-F8nb=x+eH0TRT|M_0~W~hYe`07r&pex zJAb)Qah;i$bD6c;w%4$u?e;8%&@=*QPT{bH%dT6`D$TE>yFoTLU3{K`=j}<&7QhO7 zX5`+wWlwYZJDwN`ZqR#=%3WwQs>3;6oLf7|bAv7syBN7-`2lssQBICOMw8PglW*^{ICnT6B*Nm+0pI)bxhrC?gXfzpOSG%9 zuIt##g2g?F=MgcG%h1;OsD3*b|L=B{7D~pI(N*20Y>`kr^wX_Q#z<6!_#lr8Llmo? z0SJq&MR&Kb3a*4Y@};uI`Rwx)JPLUY;+a_>3kZHc{X+3@VlEJMwMtL`u(|eQVQ}kT zoCd@HmJ0j5k&M+{lGYoh=u_4B)$IMnaM0+9Pvwg8R%UThdKfMnH+{&*5q9e}cs=T# z(!IQ!2l`JQZj#KLYkZQ7o)ar>>;DadFIX5YB36`zGdiA0vupk}C+g%25Dn|u_(W25 zn-M{7^cfA5t*eeLF%16`&iF!bCKTi#tYXGksS9&g>bb-b)e9>H5E(~Kq(xW{`+yNB zdpb*oe!#}@98EDJwfkV-ME@Mg9Emap*{3|*lIgiFvJjqt!eI{ZFQWIDej~FyM?*&- z6m@#7ddWtX7YwnIT@6_DsFuGt2T$YQkoa5j=m~jW_X2WXz_vr%JElpW<6{NSeq;d) zcc5Z}fll=I(4D8&o1e|GxRjC}*e3f_T8FZtVh~p}>$F;irm~w-#KE;4MBWSMr~w2I zg7IP7!NIG#HK>^8lfPCsPeHvS$OO4_Qi(Kl=Gn$Zj7*M%|F|Ni^N?(`<|$=g6)5!- zaEFR*fWH{-l=r=N9oRY@&vXwP1*az}i1G1HPCpiKe?qhz_RIes*PW*U`iA^d{4O&m zM!A^|=l0?ct)kaAx)E(?`PoKsQ=taYl^0;N(p>#H$BQQ=bRTeWG&PDUpqywXk0%Sg z%_m$F=%>;P!Nl$V9YZ2c4zvyRZ$FNap1&p+na#=|D7E*|;T_D48qtsp6bhpK4{oUn zN`xW0jo1KJ(j8HO7mt5pVTS$)Nnxe(#9ld}SM|C_-U+1C?4%_&_=Ol?{ z*0GVRXcNt|ymfRt2T2!sxg`0KI{llrUt3u9*HA2M@qyMQQi6u%{=j|3(=3+kn6`6R zHT{p=f*>ee?BaBKn#=Jr2=2{3KKPeqZ5YA11EISK<`Cpm+=KFNR2sZhLCsq2zvcT~ zhtB83$&JuLV4fGd{h(aJWMW**dPDGy-&=~gEA$i~E|h@`lmQ6f?@B|o z9LG;5BJ(FfGQKv8o-^N*o0+BAf5~HLfyPwmIO(1b0>h<(aLDQWVKWtV;QIX)2o*_#U7iPDefY5J!;i6+(#TUCp5u3Y(ARG7Zj__(6=Rk4!; zpDwE_xde(;Bo9!c`8cduhKl$T$c8>FBow*pB_|&4K4$M$!Tx`~1$1tc~3)2MqR}h{EY0*%WITQ~WLm%Tthkbs`;f zLXkCwr}L@c>!#p=pRCLRUW!q}(BId-O-s6AYJ+Fdt~f6+kI*Y1AYgWLp~>gj0cvkB zX6cE~0ShFO>&JKNj~WBg#a%Kt$}nx6j@t}Bbf-cb;aupD%uHg8N! zOswkW)+;zR6qU*XS|(GdNUl8`X3=JDk=NBN=ZJA@&R}3*I0VK+!VW8_H%YAMb{4{M z5?aCOV&dXVXar%4W=ucVO3w+5J}gsJkvLvZSXlSKeBKR2A^~eERD>9EfpUoYPZN%4 zrRcNh1t@{2+{=$+<*J23&o!v2-+tU9BzjQ%DReWX{BL*N%#DGMEkaO8>`ZKIkvcsK z{I^twAjd-u>mKu_8)c!(Fbjv}9RHU!i7?cbb@E2?9_w?Tn=}X z>u<`jx*yX=Y%S5$O-xK!Pqn93r8^gsSHtY1UXA}z~h(2HKCC_Su{X-ivV$dmweg7KwJD><;elJkle`tu*g3k)%ut z8Ax&RZ)D~&fRTz=4eKNA-(s!@(u~*@n;wdJjb2c@?cs3S6MC=#0b%12Dn$j8Hq~8W z_KZQd&Slpa5OE`C9ZV3%h-D6?9l;z5fa zQNhOw@-F)ep&$2F7uT<`5{40n4e;}!oMW&2%6a_+(+2ee=H}+E@bla8$ewuR?^k~@ zQ!M}Of!Lh42bogw<%m`4LC|?SwJ3eQ(J&UWb^aj59)!11fQ z!V#9-v#9?3la2ZjVKHGE%FPN>Jm;t4;zMeM)52iHP*mTK=06<|zbM34T}K;i4$+@s zKboND6Y7%6>Nb0AHHNT&@V5N`17~OKGl&#}d;9tMSx>emS}#qu-IOuc?J$jTKjE=f zXW*R&#=N+1V6}~+7Gm%dF_*?(iH$2OD{~8v;kPz$5&VL&BfU;69VZc7juoz*84t>h z9&jzOiaDk{I1cwxHup34o$~MSTUn~%h#du((8>~p6O-u^g^=3h!Jm1)2M;qa z2*`8~;ebb^WV&?EA<$3d+~L4u6dst5?6=dI_+T90ZRHdt1LT|ohTO(-qQ7@!i4yqv z*$)$-0Tkt~xZ;jmok}%{c0vXAsA@f1_=mGol`Yk`E-K!1@I zt3n)#;9=Y-;ksNOnI%8aPuM`PmMkD~&#_o+61K-O@OXH9>N$pfmOFN|I8Ll26W#L` zcW5~2W)ETmR;~fYm-ehUfawo3I8HFFVZ9IML0_`g0P_Z`qq9RGB^-laOz=k{hplSN zun{JYxhWtVGV77~yA7LX-ljW^@jN2-j#{A2TArJQT`%N(IapoQJ=MjOWBG;B=ArGc zO%uy?!@g&S1$P-H@@@mePo3^g_C`6PcVX`_8t0cQS0T$d918c5Eo5y}&Z6WpzqpQe z0V7QZeOSHUN_V%k(BY?>@Rv7NtEGZtDDu~moKth@OE%$|U; zbvb}pV1skjtki_j{>{i)Fm=I8Gf=KqU}be~*dOUvH>F@#g?FkDK`ee`^#aoiecT#R zYmq23v&Q+&332xQpr0l8I%k>?`%$U!qd_Sj)z(VzLp=Bouf?PKH8y*V$Q-*SEBxrd zPc1lXLR!ODOfb;jU*L_;$Bvbfm0}u8Bio#h`cNuM23!s{4kAudahqZvAeKZSm>6*y z`riPmePqX_2A!XF9VP{^AiIR_)*3vUvar9m(W19t2z7;3Ci?LVU+8Bx_P8o$E8`F5 zip^W;ge8=BUO5(wzAG5_7z)TVOx2Je^)NFYkn38)LkU%4K{Rl$WLyDJbCh z!{1!K8Zsx`1|?BR{`1s`RoOf+HEt#cnjGx^usiIej%0IG|7w*G!%FzNBz|?+*<5~x zQ?H5~M@i4!t6zm$*C0$f-dmih8om8xeZ2$40tUW16ix*qHiyMSTkqnhQDFAB4`JQ_ zuK?-T=36ZKwM~+G2!YT!_iX~18W70-(_({Zt~x1%wBH)Me>xkAyWls8r^~;noiVCbXeV#@0`Gs4Xy{%x$zMKUK_PT&tMi zmyaOeZ}L4+Jnar%UwM^3BjfEtYU8n9(b~Ko8^tGJ$kG$GbL`2IORcI8z<3V_*)n?6 zYUNGBL`woAuZ;#8SWXDT2s-xo+g0iLrRzoZ#0fOygCVWmA~7E86IDm5*P?umFc%a* zgUt+>Ox{{i%XzU<6xLQ+D!mE-5M%Nt81clQZm?QLKib!UE_P*%asIMZt5MQ?t2H2I zZ`vFxv(TLC)=?4W^iY=3(#$OpH=a2-Tc#RWE>zOm zc5rTI4%I*KBJ^HMw?3>NwwlkuPbdJU{)2TXkthai#gxS}*?YF=T*XLo!dfM8K z$H&Jr6taUkGCsh5eDCjP9g52I0An@1-6OKV+?3_4&LZc9|Jb!3(X9&LH*KshO#k}z z>&^LDnU&+IIbet!KxE}3BO{i{VXv6zDq4;Pc~i4W7Nx|v7c76gQ>Ev7AjjUv-yJzv zyJTcKFuzi5)Gu(<@O{{el9w7Bdx1q9Z1laiyMq(uY! zJPQGOS9zZ4*X0$oIRD`IZCC$4J3$ZLNsT`k3jZ8cO$~8T?DntuM0p-3IqiCinVG7k zr6o7PUu-ahP6q~mt2~sJ$IF|OkTB+GuIP&l4_El~%C4$n$bPyjD*+?o>gw8(aVL7g zarjX5{l1QPrN|R{KV;O+X64(GI^(*f9p){q^J_NK zfE%x;xTi3CMOB|Iz1R?Ur2|C;7$das=FKp+4AWVVpFDb*L#V@FCv7?GyRLpyv1Tyc z$6@6LjB|+jR^Yk=F`Ek1l&1~34y3WETnPBhSqCIjntiZ!<;UXCCl3+kf3zKYzsEP{ zQF|_}@K)`vve(>CCbiWDC1zXT<&EFKqW=_<`jD+1@i7dSSiJS1&bzU_%CCUyPmao} zk-W@L)!#oJllNJz@8rx1Hw|Hq=$pEEc!+QIhjm6Gn(ixg$tyPG z)l5VV&J2kfbat>FIV=d5OncF`$GnLnKBkLs7+7kKW{XgynjW$DPQN zQ_Xh%NK+fb5-X}mjH$QyO!ftxW1r+-Se^&hG95Bpd%Ju^%vABSUM{u~o0)p9yJT%N za&Y{|M@sZe9S@2LKbekCdVRgfaNsP``Ljc^-w!o)$J6@c8!Cxzrj*4x?B5Ftcq5xuDfqQ&ZiQU+UXZWXzGzyPuJqJh7rSGORe) zwKizIc^teI`UU0TBe4OlbiOquW@7`wd@7EqkzCkm9Z4`N>G)8((+EX{O{1-mwX^P; zc-7l6t|qMj&Y1opXa4%eM($N}Tiee+3|Qb&V`?jE${*4qv)oi1c)M8xq}?uWzxoE| z#4)M(?0(kV^paAzVjH~aK8OVH^oz23ASTymE=K(tnjHMXZFpd)J@q;%k7KAtarK_O zF|C38CTM3TRs#|X^gv>PSgwN)l?1-!vs!_zu9~g6JUMqYX3R#{*+OwdX?VWG%_4az z>}1qv3S$e9_ZS}@R6Jnf|l~9i(!WKtI`43}_GLV%GW$fF2vHRtH?DM=qzWEk@ z^quQWi7iHchFS&Zz&*FE^GdtvF5s>^16Wk0{Kj16D>Fz&xkBzSw;XIeRPVp0in;CV z?CxRj&IKaNGmK(oizEKlm|8{m%}dyP$YEi&9}XzcNY|%Io%Sf~w5%rYoEK+fWAhpg zz9Jz}Jn;Rb2AIT_=Q=w+{@B^)FghLQE1{wQT zZ;nqUsr0@w_oFDZp_&kOS$w=IMHnMMu)oz_;}jhNwNi}p3e|VHSaBUAEZ23ZPorxv z?e+`rRa6`%EH64Q(#BarZP-Ys%@hGZ?S5RYRa)?qz$SCiH(E(NH@Z-G^^=`nxb0IB zmlYdgd0sRJ&&O9z#=T#7^CEKC>UjPDC~ni!k#U+(5e)vzb*I)kjn}%uOh-q@wdH&X zwCVdPHn$C*B(2I-H?T$L&cm->)ZZb1zU{A->Z#pm)dBJ2-U!vYyZEO)e8JM&*Yi&1 zT~VZw;+^Dfn@poHcJRmE^RqSIZ=FP5c;m7_UR-!qfeOPcsnFn$lrkue$nH&- zJX*#fiyZ76YhXRt?w%gI|8jfG(EMi+Z1=dLu;UZq>FJomk?ZZ4yFY7nN@Zy;5Jb-6 zMXuoU|H`+>(*VvbsJ0e7D{6i2ZP`s`UzS}>O~;D# z9E{K2RMgb7z~()(60ylCEG*T zDYdSj_2{LIBZ<@7u2CXkONAZqLZapT8J*aoYf*`Rb_^$emJKugze{1E}$|Cy!&8HSL&)DS7uq(G*la0Cp(_B=qZ*)O7g zZt}R;)A)0Rm2Z}{iyxYmFzbkx2h2p4=U5h}2IxE3fO(}0B`=MQW_ zgI?6+bxR`HFJoZ@Nh%#5CWW26wQF_Ir` z9Kqjkyh9z`=Tz=Zbw+CvBI|8X{ey zT9w&6j+Zf}sLpIwb%df|AZq9+j>99Z&SskBLfT4=eV)t5&ixWB7+lOEGhc_@3XXEA zV^NQ}_&AGBzlQi&BsN;*=g6qpJLE$#oGme&Lot-YF`&QV&jWKk?u$^NBgA|eI!gGu z3h+A4;m}Wgx6iYOY;H_?j?mR*LjX}ATz{JF!_8~3wF61P*BEx5wgA~2L0|m5S7mIX zEv;q)O#}GlJOyHYQ+Qp=LXSBu&gwug-KLe$364w&mbKLnC9bXpo%zP*uY2>I_(prr zCF54a{jPfORG(`N`@a4T(3N5g6y;{oU^ZD*S|McvbCA!ll%QYX@oF~w%2=9SAp>eC zt>oFWXA7g^aI7vHR)6eOP2rrz<~3zeqqDYaD7bO+txec~>sEhKUWog~a-5lY-F)`^ zhA0B>X54GE;s8icXLXzCakY~4g8db&hBsR5D>hzHI%Ga>t{-gwu&p($h`HX69=&w-YQhkz~BNGN7}wyCd9# zcG+;@WZyAgse#LcKj}c04)L}I^vUO?ECD)E`>#$f`CX=mb<;Esy^w@=Y?>mF29Rn5MgZ`v3|5z1q5 zg!i>D3mDXDd5$XKd8cO6*sI&ID|}kd3C%*G4(Cm2~fju9Rx ziE3wxx-MJf*-tcECKqhT+vRAZr@D2J@QXw9*`rHwd5@y(pd#RrMykPQs%#J3Y_VEM zcaqcWsUA$AY{RDDvr!64+a=d9)4u)v+v;3lScc<&Rx3`GVXh9 z>Fms(ndhgDcA@&M!y+f9@(^x|eF zC}UhdcjI@oYp<>6JVNemNA9NgA3vdWdy({JKOJ7EG6E5<^e0iJ^s4_x&>1RUu-crJ zJ-NFzo%$Qlmh665R6+aZEuqL+oy8u_t?T(AK|Ybc!ITsfecflj9+2%lCCCB{54Bt% z3l**1Nl?Y5S+HK@T_>p^9f=~3_cqOyC;Vf?=R zI#r;u7hKu@`O9)TrPq@?5BR4hAmu^%KnUoR_wGI~g{3P7QuN9(tjTYoXO{C*Y zy-dh`#jT{d<>^TZ_hOI41{o_b0Qd-Evl_vKUjuMtFc3c;SO%bUFPcIaM;D~MV9>Vu zk#*}rg~x=3$Y>^F1&*NCcg6%?xq7uVEuioSVuX#A0%*eea+#JejWpeGI(i;3LY`LN zeW{HRg-$f9cV@=P+=w!dzP#0}z~i-=V5JW$UQ&0}uXzGk&cPpYV9cpIeSAim)hYNH zkL9_IGW3Rv=_s;2FgByJ;8;&xSV2iw9uIT=F|>%tn-5WYoWcp3Su-4&aK!er-9Dd; zZt{RlBVpZ3@PT^T=_g}!L5q_{_vZDx)b(&(U5whrdfDZvRkbxWHOq}9z$v$5A#`RM zpGhxtTeJ0bSPLi-bDTanr=nN_48%VCS85L$w9~Nva$^~ZWIFHtIm5yRJ*dz$5>ZFR;VveleJVI=IwfK+S06 z_;Ac={Jl>ZD}InQC+NPao}wrVEA;scbXvwAo=B>{@~6malH9P^C`nL8K7peEYqh$Z zi()pig%zl{1iYnTqzeJG6^}QF2*v84g{ygEoJYf#!#Gr;Ejp?zy2I{o6yw}sUwKip zUOZod++l}vq|dP5Lah$Hq`Y2#wNhj=3$ff(mQGkt?_i(h7M12W-7r-j&OaE6+B7K? zY~&PmCv5b?qBIoew^IApQ>&aSk?)7iS?K~6*djP^nyf)z``s2A7+3rpeh^|dgAlZm z%b9_d3UX&MY=*sO>xT==3KQhYn);L9WY%S~1Ma|mD$UrvEB)2XVxcH6a{zUK_sh1> z3yxNOOdpy4iw4+{c`9J{84d4~UJvlHTQOY?;d0^cgc58#Uw-JX@>feUNYP=F>*W<@ z$5!cC(u>xT51bU^<>lq!6~^Au!5;3tiZzv72&ciZ!du)4TPr;eE5J{J6@m|hIsp|#Ts;PVSHUIH4orSAlWa5KW?1H`BiVI3YWF5wg#b;!j_frM5$ zES1z|&&xq1Fib85n(trKIs28FxLUo=jk9&Yj}B%HzIJyXTu{L_ehl72I~?*{S7CU0 zI&|0JWz6r7M$Fl}t~7`gI-T%CEzmB8h^D@Mr{SM#YN|3mrKcmMTy?!gJ^47deo@~2 z*Nisc4N?MF+Bb6*>%s_6kZ_#V^rc7w6T+!jx5njUQ+NWy3^zc)ri7UlelVbrsd}V; z9*SFv9bHy1SijeaLJA};*Oq8?Oyy^$#`bE6ykJW_IS0iGbB~}NXQ8t1Xqc_ume-zDC zk|rS`A=Ki~x7elW-ZH={!B5ku%R~vrrp<5ac`NwYHeBk7OcUGkmP;rrRQo*7al|s0 z(k3W)S8CiY=0N&PzCsLeCmU0 zi9Y@r^AQ}hg~6bh%@L)Uw}tQJ|GZ2GGyX6og|8;$A;ZK(L{1rfzyg#$Iz15CQn?B^ z_^aac-`HlJcY?@Fc2IbAb#>2{<8^*jdHEhxZ?7&Wg0my$>sPod01E(A*P3R zF+QZ0_aIFeNVl0rk&`kMDMvlOQW`(7w{UUJs(FJQwYNx0|BvKKP2bGJQjxjj&9fk4 zA(cDtdppaYz5Qm!YadTQj{XqtEAXsoo^)M z_00NY7BHu~FVkL7C*uXv-HRn2m?w+r7V-TETh)1)U8;RQ^1d#WEq2~1k55!fENZVr z)Ki{18xt4QNWYsV&zx*VywFQ9Uhk9T`-H~zzN|RTMf+-IYx>D|Toe`8J628e<)(vZ zb%xg-Rb9Y2i&b65*$Aa|Ocr6iF*8MDErBx!1E(z;dOs*0;wBixVhd6fs3SYzwHh4p z<#S(uSli^ut2^!zFw*=0@`J4)Z!yi}?hf4m9lK}S`Ppm~iv>r0gY%?#Cb!_*+pc#1 zq0hSoaLf@2Yn8m*bXQM9CGf1nc&y){j31}r)>`9ztx4Ahr6>9Wt!v5PI-gj*v6hcFcUkBADqLB)UXb&o zXe_#m@d$QC;dLYUCCxX|y}lc{G2L`5h=)CjZhZ_vf%kK&r#^F6jx8bQXUFe5jkWqh z?VI^`%Mgipy#u*xx=-^+N#fEc$4-hv?=3!bWn77(DzoIhrSX#LLq&3st14}nPezUI zKeXWpqv(rFx)G~uT`kj*d}?szu1Ixya=6RvWh49+COJRg@Hni)bo(ZfyM_IZ$gy@3XLt7IRPo|mk7*DP{uIn+!Sf%^Z z`*7TQ`|D$VDlG?^R1=Qv*iA?dGHUlLBT)=DtK%o(vlbCOGPj+mESn^X9#@$d)GO)Na%R{Em0YY4DE5)OaYF&h z!_lmiY{o4Os^5YpB0i4h9)s>$qDoH|B6{*bQ<8O^2Luomv4fAr#Lq9qDW2qpo$QG; zVwh|qVsQ37F?f3_wB3#7NMVwDFu(Q=$r;Ll#$bfMdOs|p*eD{6%_ za9EO-TWxjN)OAJT?f zP})5zy*Qk7-52{y2xzuYP=L=thi_gblHG9rTA87!0fk@ zM*3s5m=s-j?egjs-uPDv)Zvp^9>Di)RX(eH2C^_DLPYYKJeQzONW~@dqa>Y>N|R2k zqRq$3fryW>H^x28znXQ~j9ZOsBI@sJ4{24R0TL3zNi>|@Er)bm3LX`VPHAetJi&0a z>S|?MYfPay3n=5`zWPD%SKEGCS^y3l&90 zT(-2J zP+XS2tnqz;^zO0b0WmBTpuU^331kAm29F@yf%`R?P?ab92};t+RXCl-i`rO>o~?L4 z!=I~J8C65M01xi>{4adXkMn;WCGo` zwXiPixHJaCFUzd1xw6I{+%?2tmjsp;fG|)IR^OJ6MDYUUCQcZdupUxuNDFtMfUjL& zATRh_l9tMV8IozU9Yp#p%@B+T{uFB1CvI0xf$xxE4JwijH?u!7D$ zGPL_T$tP-3l0BoZDjV-yEK{wa!<-0+>WNs|@K@=jXXz|HU!ix6sNm^{3LcS-wZK}f zjwSIW<`JQCG=Rz-XZk0safU|D*hwRJ3y~qnB2cN7rutu)}70HaL7-v2oj3+PDG-z>r zhfwH5=hk4$7veyl80*OWEGeMFyvB@7Z*gLlaPX8){-d_+P~M5~g>_|=lgq@nxJ!Oc z=Wd9QJR?%91g-7`tv<&Wxho;|8UX2;ptJ6AneZ#Qog)!L#_mcztR556tHE13^o(jy zg8{32Vw=(B=#1<(2{A{dA)JNr-6xtI-FF41^irhY&f>bK!Y}_}yBq(LXcHRE9!SS5 zPnkQC{59pwj=wi}3ghQUDc5M(u_caHg+!(mh zac|QhtJ;Re{tI9=Y=CUq0#7Cd$#sLSt!dTg3L-{wMHV>eW=5}aTmLxa+LRXkeR4wk z^c^~A%ju<_cy*`8Dx4N(+AnJz(uk%s3Noc0gTN&U`nab{lrajawYAVr^4&7tgmMy% z`sDa6$}Scx9$a}N+B!<8a{Y5E1I*AA_IOGDbYtRM3YxquGRCCriMp5xY6NaDHA{%Q zBe^RiyxC~R#H|3k$jE{`go|Ry+*{a+z;z1{Xd_RP2%Gs|*Ijn+jrRhDP-XRcL{7RS z5v-?F|K>9)35dP0hklDV0rXqEfqoE4 zew1PA!Nd9|DL(@6)>1<^v<==03Iw0$Z6rV{9j^MWm~zf^qddq5OT{SUcp}ctvX&NM z9>bp_RC#sBp0*!J_;pLSjCM$|WNWm#kyw1qAhHhQGM#Ag(9q#mjI={1Dj^RTV)9d? z({2l3rK+Bo}q3#s>`#U)Xr|cOepuelxW#_m%X{cEQR}od_=N&aus7I#G}*tD#!!Br&q#L z#bX3LJ6_@{)649RzPr6T!?|=7%;aVnAnQ2feD=g{4G@OP;z{)*snP~=u8f2L^EP&? z2tEjW;F;v6X{rMUM8(`gwje%wfcv0DZ^oI;kEVZ=aj+J{N3jCV6?QDi7J*>d-@m)gw!bS zukH7Rh&KAw=8CAy&pU#aa^ZJs+1+ZC`jHJp5CeX&QSk2Tuq;YR)MZ>QSsrPYO94K6BCWl02m*#(6` z_uhwlSZ)#EAem#&3!M3T$K>sAN|*BN9x2)O_=|AoER6b>e)w};%@_yyXA*iho+^O< zHsvw|Q%m}a-@B@2TgM4pmKcIoE+~Qct&QN7@<^Qi?YqAe?R}mC=YgM~R2;=lB8a8h zv`+ZS7N}`NzN&nUrJcEH`uxM;uI})2BM`&oi7bBK#_1_Mc8^`!5Ce_Nga0Ibs5Nrq zF@eRJt}KzEfKvv8@_q#Un;-rPy6uH~5u~J$xzf+u7C(zcJ}qLyZ6Qf95^dMfBqa0h(7vPD6TTkUoQ+n!kM5u{SXLz}OY2!W>gUw8dw z{(0``r!}q{sI)YyGuUZp#$l;Ez3_LytZhJC4-(M~pcybAQHmC@7iOP!z=E7DiO==7 zz9b5><`Q_+8e0mFN$&YzikjyX?|UwlfVw)HW-1EUOP5#=i0xQ#vV#{FB}(KmLvrh^<()Wr zjof8>G-3dQdSo$J0@C4zm4`?KtVuISPJC>T-aLjbN^d6|9tfD8k9SEN&D}ax9R(z; zjldA`8>v3IV>nx;d6GHOD!R)@(O2PjxP#?E-cwDWj@q;B+)ouIYT$7Jb^30anZt~U zVuYK1%#xVf$UR{f+daXkI>aog5omXH>@mMP;G(EoN7prP8)Y5RnUk`X@b?{mE}(?a zi98J?T+}!-FD^5xx{hyO8U-XClZuuF!(9oIJ&-CXa8PX!b?U$qet-m|8>Ui%4nqvy z>BN{NIT?V{e+M`7S|(+V6m4N!B2~F93&tn__qde%)NpOr5FaS zG*?KzB3yt6M0B{5v)6`#{f=RN>4DH_LPkqtscvkL6db-JOD}d8(LKokL!ZgAYBrhI zec=a~76bQ3Fk#{Ep=XK8JBb`LVBWw^QT0X6zv^#1R_l-s0n@CeDb;*Yl=Z5G(-!jm z|IgY0MwhQvWRE7cjnoPpFx`33L&OLD;n-y2A*c6=_kywo7>nm^ih zo)Qe{t*kt{u%6TpZfdqHu*n-G;b@|!?Yv7-?xGr`L95ZIbGx!XGRSs7VS~yWjw}JW z2H3@QW+uRJxFeax#^jPggAO_@X13tlA~fhD;B0}8;bo#fUJ|HP=o z{Lk(Fb@GO4Q1iLtJV8DrD#x&R#|Ox2;i5$_2K7F2Z4JFgt7}?B#VAO(oA~Katpi!J??;Jb0td?%LdLmp54%Ao3*F z8gbGd)8<-0$t*SmwcQ(nwZtxTg3;bTxitteIDg>-ds5Ga9NvB``g?Jr_UdGN!}j85 zo!t{Q@o{srGl0F9Q}Ot7FLxkW9$MTbUmPT{{Uvu`@t@9qs8FSNCKv7$Fxh`NE^R|Y z1xVz$NGN;7SS;l+%&|~f)n&g9Ub%d20|Qapi<=!WQPlM5imf*iNx=4BN5B1#l*d5m zIoC0B)_*My4wmC5$#dU$2sRT?rtsUPHzPj*BJ83B)PdNrjffbm9~+ojCU214$UM=w z@#cmFk8$(#!g^yTh?&DUj6cLL<6BO5Eo0#kcW;)9Y;|%wZAj;8!~^KGKSP#e#gZ6@%%X1pQ^Aex9ns}G1cEe}!5@jU*u(c+ho| z+|W$JWFT`l`yrkAC2j-NmVz*q%dn;?dQSK{5^(k2H`(c5w*R^s_FMnz#ZH~qfRnWfd{=Osqz9MRX!N7P?OAOD|Mj}b|6MTxnQA#d|h1E zDv2b09|R5KZb4UJr|IDs{ z?rtT@sRiD$yoZB0X>s6`%wKQWyrjp(u5~lygF#0XZ~^Lxh(Vj)*Ss%OvV}F;#lm!a z0=v1Ok&oxX6ui1^4ZSJt85y^F?uy&rei<0h9E8q$m2!mCv)9}poAJM5{^o~Oy;~-d z4@Q2%t2Lbg5O?VBlQWmdbb&}1&K#=XQQGzaj#$QoKh&mcc^e7>zO742ylrbe=|!w) zamVv}xz_3_;3Xg5Xm%nkXKH`;w`2cA{^nnCS8tjf@BX=ZAn@)X6`sAFP{HyTK-8^d z9X31f&&6hX{_gXg%Fd6S_=>jbsLLuj>DE5gl04U4W_zFKaSq^eRu-RQ@%?hDqEg zr}N2as^?1$G;4gBS_q;tiBmnNRwh^Ng?WV;0Ajhdmy`eBPy8%#Tq6sYW8!7H9{E>Z zZ#pNp$sFQ!ziAfujciH|ipap=uyOSLtUKH}*sATk2Db@CrgF7|0>q1+os~A|dprp#aBdO>&cDF5;0sCl zxqOICNd;`mBl@9pdvGbz5IUH>F?WmP(^9BCSI{U1$vn|E@h8HI7ffEvQ_@4(4787kzAf=xIl(XN`L5nVGPPvubdpr2UJ~=I(Gmi*@o8}u zSx-hy-oVq~&8`>z1AL+T$c4 zopM$D6k1Y*_Fu0VBy%`b79C@GzUyJl_GvVEwTz`VB5mb2k;JTuPYvyj=f1Q-ao4;A zu|@lhA3Q`d$Nf+RvJ9PvBO*Mh?jNMJ=dWFoIemF4k70x!AuSD=ZaOk#vFO!8S%KZ$jqD)8UNuyY3xq z8pIXFLDi9(2_D&V(JeGe{3sN2PDp^_Rl60+4hAh{26=aGa$$WW+y9L$i>z8X&>I;2 zt``N(K8o53vMfyFlsX#)EpbqeMmq$DSK-lUx@d8G+$-7etGa2$kh@RY02s@vs)X`6 zezw7X^Lmd$8oi=!;0(#9X{wM_{{ed6^LYxZh`&5((%gN|^B$_viB~I)$t6+N4@xD` zKK2+V(;EXu$f@w9zIWEs4d#+ZuxBuo8g{diTMe8?Gf9=#sl3_n21A@$bE{Krz5vkc$m6==q06@ z%#wIZyOkK9sL*6`X+=Y@wJqtNrP!;qi1&ohpa=@Up;$aPg1N4Ksee70q72zuI*VWOC1Fs-{3 z8CGxtKZyaMm+G|3CsKDc)FyetAZnxo{Js7Y>d#2lmNrb27F)TjA_zd?7j6+tEX|Sz z#2H5GNxX0CQDaRYVoZkpL-agg1TKgu2U^AW-8MHz8)rI*<%IGkBJ^3S)9_3p9_Nef zM~wc>v&V)3-6D2#<|fI9rD%A9gpKmq@z&mnj`Inaqt2hTvtYA z%haCCVvwYR*H&ABg1Jj@?^?0^J?^o0YXX0Q!%w zZ~{4WB!p)qV4(!Y^SX_v&YoEf3aOwxldI> zVYwM|q$<2)hgs;3QdCa};t0b43-*4dMh-`{V%Z-hN9+e*s&o)~WRlhhb8INzeC%L> zgD@;`p3sG(dlWV0LK@xjqW~!_OkP%h zGYAN`UbEJb;OY=!=m+FdKUar=xu-;%#3}>)yv?|#MIVqRA!O0pspVEDkXOXrvC^qu z6>C}xpzzHko=7G!iM`|4pr#gVIk;nhWXmb0+o<6_>Wzn%T=>%v(P}{CcDr5vn% zERxzFp&agjr%2UAjHM0(7eLR&8}c|yM31AhDQ#$foy~6b5opmKUvtt}MSE`PfLG=f z?JfEBD&x#P&PdS{TSMI802+RY_aGMT0y$$$u3Um6+*O?B*?LUZk5A!>kk6;O>k1ah zo!u{KjNyu0xQo`Xd-9{bC!iDeYI{d=;xwNGSxWRx{)6!=cFsTYDNzE70yq%k$uNy= z7h8JZ1hHzt9aJq~mB2W=(3J}vQ}F`g%XP*;vkcbEF4s_(0SU)^OCx!^|6r(dA%)8I z@hWMe2KggmQuUn1k0z1kBSChTvR=k7YXbYxMm0T6mC8a<5&i)xOSVds?j78MN0&&6 zuRR76&4BeTvdh;ON*y$@NH7rJ`uEsZFE_Mxh4ws>t*4lKcA!g`xDTrp%7yqn%+du zu%^=H?Gq)7=ydM}}cvZ#ni6ObAKl_tH1PLL{HYJd=s zE`;6^NC@9aaQDS`-|zRwdU@@-a?W|?nYrhlduAvd0)SK2|I6ys5l?>VJq1u*Y{1Qw z6?^YUYFvWl8P+`A!i-j2Z9?BB4`^`0Q%*@%L-BK@lHVMcd7snYPBenl1x13wO+S<> zoYMEMz5V~sXmnyGL4`k`${&RaKX41ZYhMTLu-}@aI$|Hj(sz;Nyj@%+d}W9`q^GRl z3CNU%b4c&WHHNPO)qO>N3 z-xZ5!NSk@(6V1LK$2D)X-pq|o!`7P|BvXoZ6f%u-6QAW)Isjd#5Irf&VhBn9<4&Xyv?EDkwa!dsXup10}0y@8+H1{V49^B5la zyMK4$>_&iH2Z@XPCKZ}H1#BO@IMp2)rdZei-0YE>eDCUy@w&pYf@-cumxN$|HI3F&mOF#SsLkmy4|#r+`#5&d zPd4^vzz%;?HL!VrpW*fNeu!wzzEV+szHG#5DZ=0D8h}vBs&_+vPxg*PFsECjdwp8L z?8v{9`^X{aX~G-YLOGR@%3qYrBUT`=XTLZ~H^m=Meus)p3g>;a@9LpTnQOi*BXGOI zE~3`d(9qYWqx&D1Vt#Gt_u*>7gnh3CMZ?q1ib}5=d}3wY2IO7jw@f)9;FArKCUH66g(NC!@|gw6 zEivp8&ZKD0jAnc$>ts8Y7T}>>Qr9j%FIU2fX4X#d-$|m}uwkIn}!? z?4gE%5YHSw!5*oF6(85q=wajqVv*Us`+pU3dG(F^i=jV{ehr|15om@18nARS=Z5Bk z!LIri`KHBi8Hd|{&jN5ZRDF1lAC-;yWeFAGnj~Gu%4@l?09*hb{t7967HHSD6$W$i zAXhiOwr?Y>(GmL_!`Uo|oQ*#HIC4*Sw>_*ee=*{hP3Ri@Abh`%8-X@XHpB!gye!8; z$|uZ%9&k4XW`m_o(TY103_IH3MN%j2KPF24&CmuOps<$n^vSYDBux3a;&W^M3VZQ&j3yPPcE^h(De5s8_O19ch)=Z zgs`sf`^s07kB^~SD}uV z?d^XYK)@iE6QIov)zFsG5Mvz->ZIm+!&fwsk)J@D3u_o8h0FRRMu4$4SUyGA>+R-m zT0ge|LGVB3anx8}uh9i?+nCIHyH(Mq@6zhw$HP1|R&Q7vc8gD}V#~-8KsO1D57gH3 zcMmLFn1DL62t%eBkPVRxIt?fU3(dhsPx$NPm9R-OT8BLPx7gPj2ko`upD+9l^?~R8 zpLN@u2fqLjZ)A?@nd(Z7TuX*F01=UT*!2BE^+gXJdp-gPe;`x_J7H^KDBG>(E!*A> zh<_0_P)HWpfs&k!%;}%hEh{lqjR`dPb%Z_-TziL=ogNp`JbY=9GZCJAJ&&EF@Z1nI40g_`IZU+CYdYMVAzgf!j zsh_j|ZB;-`RwG$wNoD%$k@wV9qgY3dcJCT^bt`nKP6c&CZZEvV6z5fq13CJz!Ll9s zuL+3ZUK@#6IR)~vyul(X^!UIDaEPhx*cTbtzl7lZt0Y+_FivF+NbHakntWud)itBI zZGDI9j6Y@;ivoX6NURzfH^;T0mckl@xQ%vHR@nfo|<&uYC2-t6@RB1s$S@n!4Vin=|%6OY=5nB;qVU zcvu3$W5Ah~@z5~@9T0+EPBIH5*&ocsE#OXc)lfNJ#M>5W))@4PHx<}oaLUCa;kdiNy>S0fHjVx zPHUR1m|n9gEAh9Va`Ns8bq!k$fvhG`OT1s-^|bD+B4&ZT!RqtpVda^(bE<5CHpSo^ z(E5Dq!7`?`F~#@bsprZ_z}iVfk-#cs#~)(HmHjDotRYz$xHTh=Uppz#Og=EU8%XALmPYt+8{*r06EjS%@E* z#BZE;n$$eG7E+Sh0W;eBSh+O#6xRn`y(D7wbY?gBEHDhY zOWH{Fayt^2ZqSKUf*q+Q6q%{;1*i4}S_~WJHM8){)o)ZE%`k1Xe>{(25fc(a6HMF*@9y}<*RTYBo^b71IZL~` zpx=GUl!VEErX7I)BJqyEOjavg2zwG05G3XhwC`7`Lk#CPznPJ^SL~qcBnN(lD&=5i z<*<7c_za!_oAL~kh^OFALdT-7Yp(Z14KS|C>{gULLpGy=;EZDc33)D86iOUL7cB7t_XXUC7^ zkA-ZMe+q;K_QuPNxuwy*g^j`%L;PSsk4XIcInosjh!OZ+<}8nE2L$}l^ZBnE75TfQ zog#Tf950$fpKXRenU z*ii|G>@G@K3(${I8si9mD>bP|&Mv>yVBT(6Yb_Z{o)c+s$! zy>y&g(iAYo2RbK_vCDulJ`;`r`<7Yn9yl?g94~|MgDKNbfY%Z}s~-O%E1i zve!w9dbBLaPgRzc=%_7I577R)P$h3jIgaiwg))lzA6H;!*y&O61>E zTqi{$s?^^kdJz;L&uJfg;}~i`VCU~!6wx#X{NDjud`_7jSJ`0kG~C$&aTu&{$_J}KNBtfPdbL=XA}*iS({wujJq{}M|7$e&Ra2%*bc4@A&F^ZMrq#-%vItL?7%Gm-U z!R7B-F@U%U-#u~VU=SNI?f{G4i8Gj5saodnX9pJ474a7!Fd7qHe{HF*e@HG{8R^@c zU+v||&*BS1f42lKIF5*!Z$#kwQ?%w?twsK&0Id(YSa|%u^J2{7D}GM+l`PtZr(+Vg zSl5qFEV-ylH1p1WX!+6tB{kC8@AlgB*qH*aTu6TcP@kS$aSTkU5kB9-o!`aDnhueb zT{B%}^|F)rT9c>A#sqUXXm3S_o5dpomh?1=4-wIOJui6n99J;I~; z?!ze~yJbQG5$KdaS+Z676x=8IYG@bG$5E z7PB9v&=v1IJ@<9d^1DAa4BOyOTxl!LJdDLhqGo;Z37^xoc0#O2Yz-(T0-NydI}C~Nx4f(HU0Ox1sd58=2JHj1EXx; z!ACsQp~FunHS;7!-#Bt!r^e_!v0GMe+VrT|W`pXf|FjJI2YT|#MUp&*Hta9MH#(?W zb6(^~+IY%mv(>WkV2@C=<50OQV3_f>uSb$6OKcFCWM;Rg3C|=h7{CRvl3~%^UI!VL z-^@%Uw$)s7ZNcjLECOs0-)#7Lo9!gDAc^{5O2hx+6r;hvBR!0yx)Qe8BR}_{*DB0E z5uU`BkRoT`R|u0`Gt7VbH(_bC@@|U!Xe-n}kY@+xAGkRdQ?H}_MWxBzSpbE)DL*|q z;kWOJLmtfPUfMHunG=UMF*mbYO{Hv=sOQ>BKCv5T!>5D1eD&0Q<6OdhH!FN{-3ka^ zW_HqjUUN3Yi6RZnP8U?)KBgsSG$Ute$?PS^qz2eifH13KIR^^4eDs@=p1S@T1?-|V z@^_~jSiimaa&*VCa{cAl)+Yx^#v*b4&0*#kt*gG540a0g*@3|sd3+%F>|r%Lk5n`) zkcRl*0mhr$@tNsc!Yd{|+Lfilz{aJXq-3~ePLvz@fmlY7jCE^S#0;fv@{#|DJw;f{ zWNbhjE9VvAQ!^0T?WKP+6YkTg8x9D5 zvo!29W{0`ivcN_uiuzLB<`vRhbzZUP>=3~5!V1!1|!iScM5!cJy z$_aP z!Ovjz^KY8lvR{2>i=3wEv@RXrs%YJTURccMiFr_^)KaaZieZ{0>F{*}NvDN}V_(@@ zd+cubrq%GzAC6K2c0v@e*BXS+95(OySdq;Uxb&Vl@q>vXK@p*m@nhvOR)k86PuN_y zc>*wt33n-)m=^yUTO(qJ|FFk9%OaIl;q%T7F_1jEu~*F3oaP!DW`<-{vxjUK@v}k9 zwCs!Y=$-Yy{r9D>7Xomlz;TtMjU|-)G#amfJ4h{`M(e>Co;U$a4(a-Pa>#RGGP=%X z4e0S(8V4i_bt6O*HTFL< zKW6N_rcHoCs~X~DQS3W_f{cp9^cr`aTADzgrls8qny?HL&*}#=+){lt@#3L+E9Z=JeZ|K9ZRg zv&kLnPY!DQcpyAnN&N4@Sry zxGrs7bzpd+36ad!>kQbiByGTJm?AFSZ)N z4=`K}8*5RMX5$?PhHGkM z8wRv;7lZN17Ft}C>6GaMY&{T2mP{F>dwmO_$^wDDW;VZN9gmD(O~i$~0qz@X9aAYTd>MWZ%YO$RLUBskO#t`d8c6|Eni*J=Zcqing(B)h1fg}#5Kxg%33h`%np@)uR zwIac^r)9Cg(?Qad!a+)}s_Fo)Jn&C!p8jD**n^o1sqkxm2R7( zToyI$YBaj;M7?2v|Mvv9!gm1s8|YSgltRD4a0jwtb1BRHLyf(gb@ z+b$Yi6{ZJdfhExL7oWB@3y3SQ*#hYUEB6ezDAIY%S1&AQd$LAU;iTa=Q))%n`27VX z7*@&9=}Mt$0I<2I3j6VnyLZIf*=AD*xJ)=}1VKh1OluHbVy@}Avu-#E)S}=()d}wi zbf<)Ho#{CviT7BF4)4)!YfT}1t6hFIh1dlESmuO}05G3KrK_YTs;bG=J7eTaLRA&~ zQuV-Im2*U7@cmI7L*H zwutvE*_TT$TxF#?tJnt`in=z10!C=#lCInf!gb4r-nY9O&+*qE6sN8n`9&`N52_}U zn@x0be;lLy`;al5X`t{YE7pxeo zcGcfrmi+F2T>vIAJ`_|u^1jC?AJCmjK2}ZZ`Lg~LbLBQV+l1Cz4LUoQ+bRxpMgVuAb7eU!Rzchc2`j-+Q*w zsQ(;iY72u=P_>4&t;uX7y2Orl3>p_!>8ig{mXa*1=x%=_3gl-U%qzwo+oq(!T%Km) zfn6z9 zA8meUKCe%)AclDl2-Vr2+8X6;S}44G`r2v_&fNdZV1?fJVlV2j8f1iW?Yl3Q%lj?` z@rWlP;r@7=5)0fAzA3Vq^-)B4qe#8LQ6P@b_G03>;GCI#`LeljTghBsxY*8}7o$aB z_?Q$N=k7P=*yym@TDGg3;6CkbAH;G8eZw|x=H!IZ5BGpy&+@lD18*9iM8Qed(~V3@ zeQ@O910J7#I_U8*DE!(isHSZBl}v7#d1F`YavcI1nR}y$*@3#9{C8NqsekwQP2^SAU=kJVa)w$3~r%ZF4{Do^yEb3RyzGEsQ61In$B7;)h z)397g1nA?ZJ|Eqmn-=ngKV%ug_4Tprwz5-keRVwksq)uzjpE&+d zugo)dKA_9SBlJ&SY=V?eK1`qewLN7f>)2+XtI&l z^D+Wur`VJ~SY(ghcTVm{=KIHm^w5&} z=MndE(IDt@K8u*LNz8SLCn6x%4-s$X4!_2W`mY7B=c{p&FY+Z43!Ai|@5D^BkhNGb zW&3$YGt{70lixhhSF_R@GJ8@(;Pi@lVkzulbX`h679!xtcXp-Zz<2d*rDuu#$B-f6 zrD3}S$EgBamDDU-=T5O&>`F&<$^=bRp8xKikbVQpQ8PXK@mYY&D%lF0FFb19#%>f} zh4rz(`T&-pr%x+eAg3?0=$`R|d3u?^XP&1^-MvWu(-u|rPebqJwi!bKf7Pzwwb2*b zm!RIcPX2*;bL=5%u%=$`gRBjWC(!EVrU z2fNC_F(kiExym$}t@`BLp@;m&I3d!GAl5xxDczy5s=(4ATgT%XRzS}#!joE3+e??f z2r*0)y3_&dDzJrwG0qx`E;Z^0IJMas6B(AIK2`c6zeig=xq^}($ zPZ!v#fo>}y_v7t*!&+lOVjQZ;6~lJ6hrU04mLgG_;K7c5hbdz{zKh^MLqE3%g(${k z)q*WnbVlZwpZO8*r4uqizW2rQ_)pOXWuiLLu)CpK62+FbS%g)PK8I~wC-WUjXw8bu zDBB0}tLCw5BjJviVK3fs%etr^V`bXAYf5=f#ob1p{QA;IqOV{GrP7sT1p%?3w?{vM z8H1Hjs_-JR=7Bn~arE0;qkwY5Z2uDlNT@d1hFc#0_4rNex#$e?SwG;T(_qVIGTSJu zEALGyOdI@Cu_Kh-*~HilyM-7oX(I5(s0u&yA9cIrq8q_u+J}Yevm0GG07kpacA+ms zMVip`d1t)@jR=W5nECmJ;cpva&ER>uV>5Iew>M2b*S6Z&Q@Q>F4gR*pcm* zek~9~nw4OwpdW};E&I@@eEjX|TIL8o;nPLUCM4VFGzxe%V^WJe;El_S(|>e#Uj_3W}>qJkeZt;}|VEQfiCt5s~HEZG@W8yDtkH&x8b41~%u~ z>oPyP7~5}hB!eSC0F+49Tb8&QQP zOP}dXo0weJ`UVcLvJ zkxX7uWbb6WbDEc@E#|^(WmI&G?3*7G;si+E0yp(S6?+L^)%vhdDxE$H$;#`O=yC9D z!*P%S2cttN4HqpQ%LJWJ63c&86-LCNhKt`+%$)}9>qZjT{Wl7w5tYv$aM9ihND`6o zMy4uktbz?C;hd(B(t-$kk((DS@=;Awl2Wc^TCyPYbA8kQ;V|UteZ=X>*z`ZgCih-W z%Qw@_4XbcL5p*eH-1K7;^kZG|HH{5duMjbM=2_B%s$TqljtcDa`F{SmLtNwhN>jA= zYfcj{EFFx%P#*k{zVEbyp4~|Jy_%8Gdj=vP;qf!nQ``BKQ7``)jZN2DOTZeKS$jd8B|KI0=I{>Pvg`_3Ojv1i50QO%?MCujWn8|~GZ4iWr$-6x5+ zs>RMLn-}d$x=M{e9}EE35BBEY*Uu+Fpw9aJ!N$0jgnzBpo+wmz^#rlDW&BpdlwW65 zV43i}ggRPpJM)0U(6AJET}a$)hm%(SGVH$elu}dG-S}GHJiC`B+e_|`A!h44-LfMd zHR|J_r_=Bjp`+-8>=`>B9ix9^`IEZCTldFlKs>94D?^3iMRTvh#ob>AAV&0c3tco&Y%@reXX#HvsD_W(}<94ZYX-iew^! zt|vsiaI3OaSCO`oEsirLPz{$=fmlJnZq`lWT<3z$ z;Zzq_THT~T8QTp;9C?@~TUeEX;+G-XexajJOE$v#&WldT_d1u3&;BI)>A!yo=sjKI z`ccxeP&Rw{!^r|F)Y;8K_-8>Cm##)vu0wzH^a|Q1Y4;YY{duZ###;^Ielr;P*6;H> z;BZhMfAfv1$J9>mXZB>jz}~@b%YH^40BH$Z9eYGJ^97{X=XO;e6QaRxbMo~$OKZVJ zr$oOO7O@(il$%U=cgkFAqr$AACiXV^MHR`+irwP3PZwc2{&gV2PXUL>wAbLG`(N%t zYdfj07cIUuS!pl1_%(+wA`yzJ-QK#UXOzsS1-nBwGv4#M&j2%H=Zk8O`tUUsyTUHk z%Wd>kW2X-Svba@!)Ma!sd~sZeeQB9E(-5Z3B<7{ND!L+bkEg+uqq$5vm$hGe;RWyR zbX@E=+D`n(5$VN3`d!U?^v-%qj(-5PP3B#ATMJ-c_AOJjwOI4jxUvl_-eJ%1`EOYc z`3NlD$+eu3yhwB6=)Ix4q;qV)eJGqiT8>3@B&gu=>|vjc|X&1~QhvZ@;j4VY3YErm|W~{M2@De4q1CUlY5y z1H-Ga_*pkhODyF4lvc>b%=HpSvIPbta5$}DxOQ@|U7>Ol3AOw0FGI3yS7Q z=8f&U8^aIh_0QQE{tqOyd;&m1I|5fv{+7uwT&I(kg0R`1^2Tr@WySGPjC_Y~uuXVxiEMKw4)iEWj`MdSqH`-TLQHfS$^SW>3zYo!? z^@j9OWpl8i1F0Ej#1Xq;KMkTG%YMu5RjcI_e)?h#V>cY@b~AYO;k-X1_-J9a`m1&L zd|-1aF#sj)x%SZq*hP2O)OEP-vuWaYdvDEG2$-SF9a%Q=+{~n)d^xs%Zp1ckV|Rgz{!QN5e*r!V5>F(&bYj-&Z$C!5 zKE^(j`9q~DFeYanlhe>kE81^vvU3?dj=x^1$4OTms#%oV)`-{}ESLfIME4C2igI2Z zT5imL6KXJI)q#h>W)jxLAe$D&p1`f5yj2Va-yKlFv&~XOL-JLbjjF!s(@>2G^%s zkLRQu5au_2rbf_6@UxcTz{FYb%=b_7pXdKlyG(;Zu)R!Xm%O(n;I|%f;3el1)X-;zkeN?GY*7sj4Og;s6*mXdv{;L@8FXMU z*lZ&$=>61`sa}~)4BBpBQ`oYbGL_91B^IEiSUq^|bw_*q3tvDg%Qu`7i>WIB151EX zQ}O(ILR)u^Er zPH{;|dTp(YRe!8CbbEVyBM3ye>WSVKHN9v2`Gnirl8nB}thZqhi$vp%CWd7-JQ>N+ ze{o!imAAY++t4)>P%f%&YDKt{8Wpsk_7r`ESM15y`;#1t0IifF17_mIaH+OYeL-tR zTx?g{^r)`}58RIsToPy8dA57o5XrMx^QKu}YPD@Wq-*2n+uaRZ_KN^}T-blG<*(lf z-y_Awbs-nJqp?wNoI;Jut3n?!C2q69UiV;db9&r;PfjO#b``b8wStI6(nQ>JR6+vNl74i|1h4_oIL$!KX0S@~^v2FIN`WCsGcd0eXeIA4#>3NP_K}W^J%)Sod z%}GHau3m^b^V8XPy_xZDLjlwPM3|&C-<<3E>TIoPv;FlXJ0)b<+1dpMI5(b{7%uDP zlD;(KB$2P4z+rD}^=pK%A;NFwwM|J*tph8@I7LOdOd3_IR-?H!+JR7*_WAR*x`~hP zz^7E-$qAk=VhYCPs6vx)06Edts)$JgGu%y$x?jfN zmw-*a{?k%dmHKsvZKX9nOY!u}-k|<|ujy3I)TD=}i=Vf003M@;4@U*ZqW2-bYK^8V z-S(`A2ywK1B^SeNU3|c&Mzb3>+m@W}<2~M_=7q z1NCR+3JD2`yNn_UPZiUamzG#G7WJn@OwRGSp!A6~7Md+n5%I zyAWQo6R!M(wYc@o3^^k9#gcJ$Qi6$M6W*A-tM*29K3+;NxV=aI{&&jxNWanp$tUYC z)f4Fq4Goiww?}OY+88vX&z1MdUbt|f4dz(K#A#fu?QI|_R3x?Vvu{t*wr`E(7~4s1 z(0IVQ3^|=B?r5}@h_U?~jCSvPZsCg-O@L7aw9$R1P5gPIiJ9Hy!T#o|3bCqoVwh&J ziruxNW_BnX>Qb5QYb@fY-$Hg}#-pc}WS-qRn8++o%W3t|y}7jKxXSF8(6S%T2GQls z&hOvpD`pv~{G@a<2BvQ0Et?AC|+KZP^TSo(%0L2Zel;`x=Px|k5?`Q-(d^ov$~hf6*Wp}dKUAh z9F0c%4WN{L^aMMa!DsJ&qC9s}QMd^91=lW*=A69A20{8Zl zeUNN!F(MRdRTC0nximr;5TY6h4-AnQHMw5AER18zKBj;FJ z=lDKk`O(yZbm2;?T`SttZa3Nt@>M4mb+eO98D#|37g5hKGtmsTCk-Wk!ct)A2+$y- zgZ=m+kJWST_{haX_lMLaGtlarGXPBR!MueqPfK_CxOR0bQqTJDt(?}Yem-80nL+Tk z)wn-volpmejpa;+g4%w*)~-se!SCiO3I8fC)r&CxP~H~tKeq523muB6FQ9$e5q3HH z(dC^##B4ksLqypg!+S%oX#pvAw!&02=f~7jn201`@ z@n$j7gtbgpk;35y>>{&8&td73pV8EeM~)zIn_RDnPdk`9LSA^ zR|0Fodn4T&x$IrL5=E)(7nKmAqN3|!jd0kK?tt@xRijBXTimn7Z(Z(H&eJXLru)1l z?-g^s#<&c;kBeih;n&mCTfYfhDjxImtCy&SwdFJ(Ke(D)Bp`5-#1w_X+5@$Ik7WaQ%h^`o!B;xEv@TIovKAmRh8sqh1>a`&%<-l(m4BUO82o0q1zvNo6NW< zxikUdl3NqzOG=jeHGlj-s%@+nc`qQFG+>u(Xq%xR)5b{UhG!Fw^5U2CSv405SFVGpQ1CMFbw75TI>;k6stsR5p{U``(Zs=yr%ZzrV+1lFa*k!hN z*=w&4x^nX78uM>S9*H5z$N@fA|8`93#1RK^MbLy|aPF3#6p1)F2*}y|1M)Pfki?qw za_(m+4zk6xDj1Ixn+y2^0u}wl+`Y*l_VzBAI78GMQ9UdW|01aw`I{7z2Om4c@f%*& zcoHTa)pQREt=`(%X|dJT*5=ycF#5r3MW2(hmaE+5PB{h4uvu8R4c20XT@)}P)Z353 zGhEo*4Njam(de$6Co(C?&CPA3k!P^!`Slcam71-s?Tb8kPAa4DRlx*`(0kGeqe~>> zC*F?&C}gyQe7sevwi@5<)CS3%!MAYm0we`<9tc3EX>I{bUhy*~7d{j!AFz2^IVUgq zq$$nK2l{;M@<+R&>xgh2liPg#-hPJFYP^0s-Z?-a?Rn~<*Z2+UUnUg}!@$y2zwvJj z9CXr-5cb49TH}3m6$C9578H}P(;iX`Bxv4gdgV;9+}a;Mf4&E#SC=?pi^(gn)j1jG zsaJ&M31Z#{X)htA%gaB1zVjIOQUrD;Gl#y)*%k@dO0Y~mr$%7yNVBhM%S~+h5ECP6 zs@Zs*4x|K|jHa1G(9vCanSITfDmV zkz!itbdmeDMK<$}54+s0i!-=pi?G9LS=txCBX(_k{q(G|-$qt-4so&wW3p=7VEW9z z)VYMJtufleB`tw>iZS^{B)F#ZsZd`RwFTG=b;4g}X*=|7U~?H-x29mV&d0q*o#;2A zRxSQN0+QN^d$NhrY7ws-e)o;jSDeZ+t14#uP6jfj zWo0SX;ijm2BLnbw{A_#7t&J2wQWPs{JEY|5S~4&&K-rd=Xq?*yC*v|8`a~rq$z<^3 z+v*afKP1a;NPNO~e7oc$RiIqtPHE~PV!HeuzAzNF_|wV$f&J!k`NW_5tPbbLFWl08 z!!^p)AM|D4YOK7yvr{#@tB6HM7u1~s5&)v&`scpBC%wH%=roMyRq9i!g@3UI;z$Ko z385!2+RPk`l;25KTPMOMdA{$R)inZ-+GX1P{E?_U#LBcJA>>azpkHy>7dVK$ocU0fVDh`1hc9Rvc)?r-&9G0|lR z6qUdN@QLa{@*pHBmRIi@z!g{s+N{6ieE0f#Ggq`+E>*R)^-J)L8LqPtWol|`5R_YV zRI1Sn`5>CM$;P16w=Z$|hSf7+;^U{H-UQ{{x%UY~QN%^ZZ96-}+;OT2NTlBNXKI|B zf3-X}T4tM`o7I0m6L1IZ7HoG$%WIPMlvY5s&?2)mQ2JQvf8l(2K)l;6U-NS4CxJUF!rV?aBIR zY|it$v35o!a_Wf)Du*Em*UGTJ%=XxtSyiBaHxA>F1MloCcv{{;Dhd`0_dmxpQlp<8 zk`BM&Vv%!~e5UTGNvevIk;dOXBOv&^l~7wFx|WOSmQ69OYHEAx)VRCkv-jcG+TKf2 zR905rHC>DF+e&>k^N2NUn2Xld5!hkR(-F(Zy!owh=MB0E9A<4=2wSqj?7Fhv*7ILV z^DWumWU$|VZdM%zLe5Nn5!xfDTq77 zqNMOMsT>fOcFr5O$*gW#cztFoW)`eUZ>dwh3+t$Lw`FWxIdORtGtx*EqBs~)md9@6 zwcO@1f}?9|jCB^8-xL2Dz8$!6eDl~~joqd_R}u31&zE({D^-6s8kvRRx82YY)-p+^ z&ig*IUc!spL5eo%*`I7bhW-|l$V`}E9_6I~&u6s;ZWoV+GOlMo7ME$Pg~6?tQ?@3p z2IRH#&TBca!}m~$xQz{iww~E#-LX3g7e`U3*^(aFXkYw}vb?2Z1bfg057~-&efiu4tzYmsz|jm`rgzm2Uo9n~HU` z*y;5VOrFb6yXdEh1z7@#DH6#+^7`a!X+{T>iPUI(&^I7mX!(h_SGDX+E43j9bqaW4 z17A%8xS-1Tm==GJ!_8QcuIJ{7^Z_E9lLG(%IKlpEUtrZp*b&?nVhz@i!b0Z|a6cJC zq@0SvhPhO1w!J-hK`C)cIDyinnd1z=5^lS?)eeXz)o!638l?Fk>-5{mPnwqOle41jx z6ab+S=`rD}QwIR5-p8(@5?av;It502DslWVz(2}bT3UjA`COld1_#>;{I+jZ0^n+C ztWtAhb5jwmj}DT0>!1%yW>D)xNfEqyM@KJjHLg5j&=cL&D<2d?Hx2ccE`*=Y-7mHu z-M6C8gO%bpIs{H~tXo(-;67&q7BSJmTV!%5hEt~QW$!xmZ}xx$fTS|cYjf@d{%ncM zD$K-wp>`lIb-x#ygGwn}7IB|!II+9dNShAeHKW-`MYc#F4k&qelyAk`Dr9SnYDn!C zWQGZ+j8mYrE!)yzl`d0HQ1tsyQ7J^j5@?@wMQghh72S`k6Zg4!L6< zK&qE+I8Lm&%deGy=q+Vj5k>8^N9d}^BGaWmF34MLigb9d4 zG_S^o`r>Pm+}iVPMIl89>ZSY!qv=BAPywTMD}3QSB(b*uo01(j<%Kn$=h(DA`zzYM zI$d#)^7-IoZ0AAmE3x}@e_s+oic}_%KMS3JVp5}F3(G9+*zNwJtG~eWrDNi^bJ7C~ zg67iXjEjLG3sv$C0|}82l7c=l^ajPGXAF*4_YKF3o$KnKyKzFK?J0wTfKkmQ4Y4GU zf8Ek}t^QB}m_1aG(8QoY+14CH!}^wc(dORrcrEm0t<27_891ufJpx8%D)x;$I8+$_! zzugVJ5w|M8cPmQSb zDnZn)KIxnMdlo<~JYRPAr&Vod42@pNjlJWP{ra)vB>a-a=d@!HLD=!EHps(>`sc*g z?I9fxBnyI~?SPa%4CYm34K$056x(*OY+{FgoNO*V-9JFp*-;^Yxd;sR6Q5Ip#!2uj zd41Y^Zgf!SU6rH`&kHCEm{5mlokhF+aC>=v)?K-c;8uR|mw@p8>7NGyl?T}KdXW_W zBsT$641l!hgEcxW#n~b8P>}>M(Xmu(9?iVauM%VL>Au*mMCT_La;FxpNYE-N4w^Vt zHwQ9B!9=-}rr+jj0fUb^u?!Ofx{6sH=xt(K`cWoh4Mc z20)pe<1_;}xZYGnh4S9Ww~E4DNmAk_l%{G*L$aZ6`^a-{=q5cigP~q8h?LqKFfeA2 z6PzdY1STnI{}i?p|9SMyThub4azYU?mXERenz)v;jMr{sLpo`@gNOzUl~^D?0s9a_ zxW^W6e5-a}j+*U=YsJqv#w}tN?~EdE4n_nM&z`aa!}vtIurnCjqQ^zGQ1wS05xeD* z{^({%hiQk7jdvHZD!ufI;c1s$!O)IoQVmbAm!fNV-(}Cc+#oOPIxSvmz_t#wi#^6i z8xnRN>EbHjX{QjoBuLe?Fn6(jg7IzC<>%nY>f3kz9$on=q5F(+6pK0Hyx%d_{h!e{I~wB@jKShLO77bgN@c4X}O&c)EONyiIb zE~~2B8MQ4o?}+`eO@-puGE0=p14&xbm$HHSI~jDX_?a^ELkzwMaRg&`f+FawR^?YF z@z^_tNb9t!jASkxpy#TbVOrL&rDvRo;CqgwweNKO`{H#LswjuOi|zQRsOO#%&0GDu7)Td z-kk;QAuv5L|1&M$x+8X5#Wm>Zbuys%>#x&4PY!P%fZh6NL+9t`Pj(Rg363q-KK7Jv zp>8C!@V_HOcNMLgY1?Pfnxjurm|pK zl}5JgYP2(3Y}F(#+RYzYVr2B5&-^;I+C9z#iDY9jgB9JIaQQaydiX@6&>eqkcdm_bS^6^ zYi6*@iwVD$r%g-8HKaw+UfJvx6!JkDo>*qH_hdymqL<_`Dp~e)aI(i!pq(=56_k7nRlp{Yz{h z8CeY9dpO1&l;bqxy2RtQyx93SdQKW*!T3VS$3fFWq8)MBRh>#r*yq@cpIDK{YUjww zRXi)j-d#)afcS%)c-8_b#b7e?2_!mJ4kxIZXfsqO)?eeYTqHQ2c(8eV7Qi__^}E24_u8O=%v9n>^*&*$*!>Gb=CzL!Q-m zI=_>)Q#`hV8kp|qe6l>Riz;oMgbhz`Ps52$o4pR2dU{Nu5A0B?a~0V+Iq9E1U3Ez3 zxNKWoR#t4kV8Bo;>V^9j$+5n9x0X~^Z39@rm%UK*UcdPW1YUr2TaemkNviMbGtkqH zy~Qn%kXFnVeIqT}Utj}S7M>!QhLH?=Dh@{j>A(5l6uGoe?@l1Afa3)N_U0TUptV|n zVpp+F9c6?=dW8iwi?F#bdrU1@U#lLP{gxc&H7D;#@DJ=uTYNsA_$?a4*j_eD^jX(mZ%lyO6$~>r1}^$`yFQebXV&SwC>l_{ZbwWq@5U> zzoP7TznSvb=7jWrOVqjf#Z;yZh3V~wk3XI*qE%H?nYG9MD)L$Be~La7FxkjV%9Su;aEsWKOxmCD7cjN4iM`YO20 z7WpTK2!6Y|y87kJWDn@TU<>w#TOa2%sfL~GHJ5;s-Cx#Kyb?nx^7!ZV=%#Ax?H5fB zUf7RUu|fyQnScTZ{BN=%9hH=mh3H3R`x!qB*`e-F(VR=y~okAw>Z{8Hiu(7IL7Zj=>5^R_wSz`JsvOR zb-(WWx~}K-yq?!}3pL3a^}aGn-$hk#NSUFFMXy_oi(pu7n0hzNnhPJ*f9wAc_+2rR zTU9;m8dQs_;eA*9rq!jzVjn|GKc913&LElU{|8y71H*CA-24g;?_~T}MO9tHcYjX4 z6VSxRr8>0}#Qd=nztF>dpMp&)UFhGuij@6Pwb0e=ls53G={u{Qlx$jSGmYX*4dC^G z7(gOXj=DDqWa`b}Q{N+30Y!t?R9yPE9zD{dZQZcjo3fkW-W%hd$lc%0o$SjuTRtD; zEV`>v#8ON-=X$0-9nb*;siOnA4n_6}NkAhKLNCe1$;o+JTUN@j&_e0!Mu`#zMK9Z( z&hh{YiU%>D&BeJEqM;l)H36jbQ6MKdH``n8IE`nAEq>^s514P|yslsmPtVC}%y9TF z7E=$Gc!0u1rj;Sb_COXcz!w_e|0BcL_AXm-8jh{$b;(*i};xp$8BR?!y4J+T_NUOn6Qpw~*0zTvGEOk3tObYC%B(gFJt$ z7pbF-R1_W^ z&ed+m74=wXO8$Y_3z^vTK&iN`rMc7_Q_Uf=lucA4w|r^9PmDl>#Fhy}_YkE3GhmZ&0*0c8z;kg;P+>7q-h-KOnaZ zk9GPv4BVa9_s?-J4>Uh6K$pVY@BTeFfmlR+4XCa@rl+&WmGxc~=W^agE<&i>^+_l> z!lsja41u->T_l+22B=$xa=qY%v}r#UBUZjAE1N1l+ug>@vybK;oc0T(%u6Lps)}Gi zisz$RM`y+JJOLdA`5Fd8@v9^ITttYMKmTwqsAuLm)D6D{mx^}>uXIKir$kRMBJ7aG zK1!i)o`}thEPm(fnr=W1Hczs)KfI4Jkb!E&%?jq|N3g1Z_Nfv5@*A8BQ>O~iTfg9Y zdlhuqdLuBDS6YQdWq7pibG6s|^;*%hl>avx5i*25#}g}8{zB!?^?&^4&bCzX5Kk(T zzXU#QJ8cj5}=iof#ho4*vVLQESm-Ov>-2JAS?AO0w~;AHij!*xf8{* z12gTa*Q2tH;1D@eK!i93eE+ar>5S3@>XJ~)n*|9@`*F{*ndY8K4(&dzrr+PJUjaF8 z5SrC`puk+A!{W)T@y(HEHk5$XI}kP$DACSplgBMQ8tjIg!=!p+$TMtcQWGSXzmtEl zpKa)LDcB_}>>A(h423X51#O7Mxsbh!P?%Lx_^y3;s7+D~6VF3D7m6_@QqD_qoB6bz z6ZEdYu9z8s8s5cIh)ypgRNAYwzV}dl%ySM88v+KIVY5D#w(CDbqH;?XjGApYvJ>eG zot@Dt1Gd=!-h3TkQ>%!@5Muc{%^Sr%_31QcDO>)4&i=Vc7Lj|YxE z?5w{-R>}k09_MpTt?&RqA~jb&E+>+@6VjeTqh~=e#&{p+U;IuVsDBC(=r?UlBSB>d zN6f%Hx^mQ{%hYv~_8QJ7 zospUDVSZ@C#L=%^+Os*=lV#rX`Jn>_CQrf3M2D(z)-XUSlZQJ0L{?Q*CENalAR(4e ztKMv)fuGESAR#{e^ntAIUKV^iYm=}AT_dC0`|f@C?5f;?WqGk~xr4c}bq@F`W>!{N zN-q7FqMjSMRTrv;ZQr}y^v-i$a}K$$T;dFDp&!p9*WF$rQy;;cGBwzR?3V2C@2h4T zruRV(7g`heAnTmbQ9q)_Q^|5?O^Un`Z*U5An?KRAq6Pb+$X zg0lN&FzV4jV>2)Pvq621c3TcXpKoVhZi!bD0Kv{g=DNSbFPSX;vi>MHo;~4Txv28V z`gwQWukHw-Swys!?rOx63Uabs7Vk_DI381QMO}3%AjG0@J3sR;6@klj6?3GPTDrX% zs5i)dE_~2KgrX|o)jHhVO;JPhDNH>T&_c0dytvWNjhWlwKt{#<;pzL&EYVM6cjZf> z+jN& zT~awo`($Y{=OeCibuOu#ufwUK`bmStE>`b-<88sr*J`f=USBwn=`-gox>Z+I!*xBd zEk6dUA%m!bkeY?o#(qtmkQf){l$+I`5(P@RmQtXXF(qQTC?kDBbLFzbRh} ziMA1ic$GOdUdsaWTX|-eBjrS^iNNJ{O zqMpHnY^dn{H*MQnzD`);tX7@^r`an+vW+!TuP8-7G;hiiQ zT&0#o|EC)S9EY?poKRxTwBzFO;}hfM*IYXIv3Pa3BLH1;Lyh#Vjr-4`L3BV-;-a_5B&xkY^+-oq18kfHY@cP++6(?es<~`>yNeo6`6jY&lx4Y zEa&a`d}ykwiaGLA_sUaH8$)>l-{v;+^D5HY%7HkyT)6bGzwh|wF%;v=F(~ICUJM_P z9JS0O{DK%cEft?N!&JK0GIqSGV>TH^`&o_pMxs!_ z=x+(_?**AbtlDeNb2u5W@!IiNr-8J}_Q~l@|7NOYpMxsNSa&M0i77CjEMF<4x;_~j z{G-b5az|`FTmfkCfSOC^!-%?Z`$>#fo~*=>uo-G*!0oxpp*0fr`1z)#H6{Soq%M+?c<9V+v0{Q8oP4dW|%+;*qq<;thi87w%isl0!d zs7L4aL6yh&CVo`Z)IUu3|1o+96I~1U1FcHqBd8B(dk6qkpaYZ#_Zezu{)dej5l?NJ zstr-=y#}R~Z{C1jTbsCodd=48{mIsgEA#25ouo-Tt=1=ZjZwJT9x~%PbL(z|RiFD< z(m4n%fC*9U!O+P8QKZ9;bfJcT)4*?<6lZlu8)p0lCvzX8@5&EIUpwrP1Uy?D)mm4SdOxmmsl5A$0W@_c-9wHS-2+q<*ss%> zI=IO6{^G4?6>1XO1%kf{sI9XQmKfcv+dkr-E}h5ZE|-XEfm#6pn;!JX&-e2sj==hBPM}-%+D&rO0 zg1j6zULQb}qd(Xgl2b`h4YE^Eja2&D&pZF~YCqEovtmxkM zhqwvmciwsM?zPOs`Y>B6T1NZ6MjurXYx9hbL1p+qeY+Og87x7EKdM+(TNW&xExwMn zzdbD=+)!s0DE(IYO2ZOPkaI~bxu#sNP2cWfLI8_N1yPJgjx(Zo9rVCv86kq91qq>) zjah@u%uGnb?=}Qeg}0^lCHhKiQ>rZ5{5HMyNrxouz)2-{X|m+tK=%qwvT_%L1;qW; z;T?ttsP>H7pg$cs*bZ0>{+OT=gA4t$isk_z{rcGG)MNX2K+o{qjji&_jn?;Q4-X@Y zqOmc>*wASjf9*xP4Z3LzKl5JTWwq6^tl!Gw6Qxt58kuOY<*%7FQ7KH8b@%I*1a`LADE3tkGEmAD3-IkLVx< zmS-EfSvyAi9y*}=j0v+N#EM@3`6if!a-*is)~>0mB|HUp!dj6*o9PrNQHAMk%$F$v z0$h*ycWfo#nozyXiCSo!=+(5Dl}n&GfWwW1<}9Vkrq_XJzSxqlg6%EAnO z6L6T&@1CGz^Td5bb$!r(I1Jk8LJTaUY9t)LtmPID_z0yF#iA>s0D%c|iWp^F{PZf4 zGejD)T%J{nP5eT+6L$^IjLu)%duO#(xc$|VuybVvc5*3JbY(Vho3ZEk`yLhxz@|HmrT?~ZET!L?SN9f9UcD7cn*x|ro_L{0E~Ml5wj9E!Hz*T%>&_SOhvO5C&r zC@Ec0&=uG)L>Q8P542i_+BM;j8qXj>jhK(@Mv`p=gx0+e9yM1HZh2XI3q^eCx z25JPBaB-qFa=m%wV?hk5ux?_%MAB!q*LFW$8>Xj5Zb!QNv>o*G9lq_6`+2|wg2&IY zLapC!7kAcTr2PiZ8gz9|UL(Vj4t}F+VC{Zfneb%s-l{-ekULM(TWaS6C(Rf@BAj4r{PV+JDY^rxGMkYEsu+$I0ML_XayF1;I)Q%SBB==%8*rezo*tMzZ zY;n8i*G;CQ=~vqoriWHt!uv}4(J}@BUXTj4&(tFS7R!!DHS!zaSbg8_62XDH-w)Da zY6D3)yPQf{`DcZOAMTr2RvEt}cwAYUXvo~$>?qskYkK5wjf}4YgQ)sYh|rr{>SuZK z6U&3cs65RF5X8Z`vS@&muYxGL4g305v!$u6UdhSIYIX%amFFAQoRLDV++#HA*`%wC zc)%%bIFn3Psyvj7gY~h`?*Q@Lwg_V7lZc_d?6?FZUuOfNGhI^qS1Tu>;&Xt2!+l%0 z{=kpU_|$ryX_ciuo9Em~`2%w2BSZ$x(=wKy4X2o##|3$SFr{S~T^#(!s@Kihs<}R; z0z%(*zOK~)e%WDlJk=S8cn*DRi~t^1MIa8paUcbu=$YW08w=RuhwL2b_V7f|!Qhr{ zjd*p{-PtqtQQBYvfxxDJEty|Ykua{yreOS?e;^ZqmD2v2yah7X3^N0M!DE#q{oDm( z3RBd#|Jx(wCw~Fx0Ny9}pZ#M$Ue!M2o6~~Qt}9edNR+*uGu5CC#+qAr-@6B!n0|D( zSmyBm@nfeVr1ihWLBz>H>99!SAXy|k14?WnD^qr%Kby2hO8O4qo8M*b16bmi3^KK^ zT%V_wcJtscX%k)yHR}3PF-=V#aKiKijjjhd4pDP3k+J7JRZFzisZ^bcBmKdmxF;Ob zQejjtI8*1`zY||EbQQ2hga3sKFW^_X?uX~ey7>J*SO-T;F~#@t4$pFbC3f`YXCpOd zespx~I<9L$IJ#6%E&ebyQNM#5%`=db_lRHeXe|LVy*=iuj%eL19LNrK^|a>1#mw5b z7jX#}A42cm^c`5_-ulCfGUSfA`I;RE(NFKaxZeHAR)dRPqHoqsBtG~;8Y~AZ{8!>G z7ke~pS+?hX(%f;t@nL#LD!DgXwRZoXY72|DFY5sL8O7COw!#o5au@@rtkeR;%IK%L zY;_G7?h(X}4_$9PbaOm4cn<`K?e}-FE6bfC!Y;qsHtEQ6aL?G;^7frl%cpwId6Sri zvm%*|gUeAKVkW#mPRE}fUT0&hYw}1eXrPa zpH9++lsihW2tMi^s>q*IC>@w=cHHsi`TocBje{2^X9l*8|D}uLkF%9n%gOg%P1e=O zseUIQsZ)R)XzJH)7_B<;T|ng>PK=NuCVmMT5W5T3vBC9o#h?$i zxrY{{9ugq+;4vcr<4>62`=4(EHBL}cwQ_|lsvbZ;8p=Y1Mk%L3FK}PR!}>a<);N5% z2}BOYcTetyb>i^I`^#Sj5+5uF53O>?u67G^kK6PSlyVjhbq(Kec5Sgl{!QZYumiZL zLd++6m>nH}uM>V6WMv2AfRyp_v3csEmo}F8VnYTOOcz>tRrGe#!=JnoIov+Q5-H$2 z2Z%iF3Tamm0_c%>Fmy{ihvveF>sz(gk*`2Rv4GX~4AMF*%UF#GsOBc@tv?`=AtxzM zuPYTYe=^nBP8YtpH3cK?M2V5(nVpY$qxRa<1N(8H$}aaN{oa4*^8#@LjK+fdPyMk2 zuchD;mag>40pf11*ty>I15mi~!Mbr7KGlPaI+~N9t~lx^!?>{+fQz)t750Tg1C4Co zqh5iuyhY3pL)z0I32{-_M&2nz-1ep)tD)*FtAq`{Jc3Qi(1Vk70MgOVl8NExF9l0| zW&2)X@^hBLMS0uV86eEIGW$ZW9rZ%o1Sn4PKP2&o2Z#-KfX2G`|9Jh7gV{ZNf%jA6 zzS6yXdaCW4JG}ayfB$<0y!`gB2Z2S3m_)3D?fYJ}LyYGm=`OwOVKA>`VyQo!qBq3~ z{MWTJ1h{gJgR|$-?ZB5-NrIC`LbBwJUtlDpYN;JZ`z67@L5)4NuZPAg~vI&~sEohVZd(>SmJ`R!G&f^a{ z?qf5rb@UA|Az9?v9f&f!SC$W9i7<{UBTv3r%fk%ehH#|Pg8(U7fSEp zN7wVyQ5G(1_m>1vHIYCSKoTejyBI^}#)FH21?CHQ?oX_qnBLiYGL?&+&c;o9_&_6c zSZgbJq4qk5^e>pmB8S_m=O12Spaox-|Ke32WcXqj(5g&bvvjrv|b> z;mZ((>_h&6hZDa`>1#ji?7ae89mrU98+@@`USgwRM+8{T*SRVTZ{5q~;uM$go zS-k3z;Z8h_11w}}h&I!8s!GB=Q0=v>>4(2!`e`tnZ}T=m|DUvvzZBd(CU!}T&*a(! z*@r`Ngk+WPnj@E8p)^0yLFiB<_Y2M&aV41I(W-UC>2I!mu)teekFous< zV0a<_a6n(T@@Vx7ffW^aq_LdBkJPh)Gtr z6mb`xNd5AM^-u)ojpoP%i?z*bAIm==o`Hs}`%>+>HYm!bES0FU=&zEJyX#-#PATZp zJ!EBWWBV?_fI9U5|HHd)a{+`LN6F({(Z6N{N`$4j+H&nsu+w*+b&!wTxL7sqw=-dbF$huiDkJ&cJs zyo*aLZjZX$iFR@lCi;%PBfnt$^Az@-h+Sz=u~5BOF$PMPNz`=6g~{FX`)I-3Y!FH* zZ~mms7&GtPF0Z3t5ddWt(qe-Y?1 zK#Uiy51Ew!inpZY=SX>qsv&Rcm#0^G$??J&lLO}n${j!UF<+=>-4|2BfnplR@d|5* zvVrY8?|Rg~BIsi);t9b?Fqv4Cj;D=MPl?x1FJv*X6Fl5nF=J=}3Fvz>C8HEsUEE5krR;ZCl(pZ6tKx z+Y!@2DzmvLy5lVV{gGJ-tXD{qz}?XtS+G@5*0+@$&}$+MQaVkosm9)b)=~;L%#w*K zs<-D2@G;h>Y=<~XT{jvd=UikMOC&MTq3XraIb=9%(;KiI|eOy3a8t+|q${U79|T~-#~?N#)yntdCRB%iCB^L8tf zIhSNp;CaBVre)X0TDeZbz1_a)7owT<+oZ{vkZE$BcN^=9YI=ivRQOa}Hmun}9`^7x zWb_4Sh^FZ13_xhKh3~|_{es-aK=!NFnq^l@{ra0f>Mg8HmW#f?n*G6=9K2H9J>anI zP14T);jkYb#n$$YM=XOT`(M(7L-{ekgR5&=4%>m$Ps&)Ah=aW~(j3?KJG&$F!IiX# z$cE@NrDlC-|Jk839rWBnG$3myn-*LSYY<@$cp_sMlzq5d!vqW2{TkA&4` zP%|0HeLUE4kdm1mreu{=s(SB)@yKr^%cFp+Bdp0}_+@tj<8($e6zQ<=0T+;*0rb}! z@cf7lxjYH|kL5kA-AdZGw~4Mho?@Qxiy*VKUU+94{+dGAU%c;a_B@3fF305Ej{Y5G zL4AukB^or_wOYvlnn7u;^9}|bj;pNPwbhSqZU15yH__6g`M-6|EiEbw%K5;{Ez1$S z`yQXfU7Hu)z5MaDq-PtjM&?jbr)(a2{cMi5d`@0mBb5`y7{~<_b|~rg<2TwL!}S48 zEPeqj$Fs$Ad+rri=F^#?!w7!dwfWZGA_S=6+-v?nGPBGZ&x$Jy25=F!!{_y%5f-|+ zo&$mYj#>%gIsBd2P@z+iPg$GrGf}R5T z{@JqX%V;kb)U`tE4c9|hYICzV^Tod|`nR)|Ck~T}XQt>mzCFes#G9v{-wS#6h?IMt zF(c}USa!>I%Bjcqr}T#nb)*m3Rz3wBFSZ`G`|{R)&6CBCy4;^(b!LMGS+ik4KavJm z){+A%4scH(0L54LUKb%v&$CJxd;lLRZ!7PaFXm=Uzp*j!gfcpEvgw<`>NO@L{Pnqh zkb9AHC4)NhhAiWTiW2Cq9C=|>zcCaHU8wNfp42p81;u#!!*@8gF>ha-P${cF`HtoI zLKp-HhWm{;g`SFz1~z%M@&OnZ^V zJ%>&167jjamTl?a!Q@FG9J?a!BMaGo_+o9X zKM0DrD3p!3>1vQAf+*dLGFcz07N{RaK4#KOhPHG|pe@28jrJHeh?`l5s#( z+}i=^C0Eu|g?Rj@HwK&FcN!#Yn_td8Ie8R94>m?!<9~7Vo9sg2zLaGDIIgi8YF6-h zCq3Ne&Ef1$LPaXSwKZWDzZ}(G&UX6wf@;=**KoRv2#>~}0?svHwyzK7*V*!Cg{>tA z6DDFV_}%ECbx_N_x3_*{zSY-A)-&F^1^DUqL@3LRCP#C7`?iQHLZbH=?i(wnl41AH z;mQKE70cSy52E!RjVyZ{6>0x45HVi>Zb*IW(~liAhOhy=-EH4a@0djiuQR9$Jp?Dh zO*{5UZWWuJ0~ii{G=~Y7&?Z%ARKA7BfD6Ki6Ppp z`McHpMQcaglKe%tMdRR|?nPU9I9$t<)^WtDFGf!Yl)O@%ZM^w>yg~JU&A;i(`B9{c zjtrg?}(*3}JCcY)qDttUx*GF7hmH3d1KJJqLLPmtyThh4dG^Cr=zF))&kR)_?iG zkewZ2TBuOk$QQ5~0z{B)Y~hfd#KknC@2uPcgKP)+LDvDDSs<@%y--E9S5pg*+Cq7V zubB;C^J0L_L>(A^HL0y7EepQ-T`5^ch-oXHySZM!ap;btXBwQyNdZ|m(pF+Qp1%bG z>6d^YYu>-o9La~9jA>s#uvLYZTTk!@Ipa&F30iNX)UpC0Y z1p&ph+Rjzi35=0|oJvx)@sEuu{iHE0B1Vf7LK~!Ylf-1bXvn6UV-cvOxb1FSRARP} zW{4zMvwQJ;s9fC|+)DT0=04@uxw{Eyah?hr|M2#rru@iU10YpIvij>o9;SU@E6;~y@3!#%KRZ`_uu$iOf_Nqlrd_yFv(j9=p@6nkJzIiWc&be9XJgVq{73;K8<+|O>$B}cF2@xQBh`KA0MlS#jC7T|vtI;9i{gBm z&ps&sme6+t?Kog_@;&J3G9#I$A|6ehRrZU*y})gIJn0tT$G3%_a(D^2WyW zu7iS7M&G&;Rgy=VoZ+x8s75N6q*3xPhJD4`{`Dak@Hr$Z5#Pgq-D!8-ACc+<->H59 zXNh{bfLc+j5vvy>*S`8U1dstxy3iBZg{iV#TLY*)yZ^@(j7V{+n4y?xPZr)v; z-h)qJ@y?fpxH>;o+9jKt+Cy>;9qXdeil3M#0!kXA;Fm_1woq7pDB6Qxrx?max4z$ifYr?G>tUHc5!3TeijgYm4tgOZ|xUzSB$o**rI3z!7m zkQcC3f_~-*s+M;5?%9rM6`m`lH9y>`7F9ndK9MQq@R<@@cgWcJ0nhZ{xotU&Sp-vd z!>((UHp6D)M36pT;$d6vkgP3}ws<~okBWB1^_JOLmxs{?E|E=o(U0Uy*atq`ZHi#i zOM9d~L==Yqo&nrt>^Q9SU7^Zr3s3IZshg`b(_!cBN4&ZDBx28Bel+yI9g4{*6TUIV@y_KneyHe ztTo@;M&WbMKZRn4k`}$~N`JI>?tVqn2-deD%Vvel_ICEV!nVxu(9+g{e{q0exhDv| zZ{2@)$W{rs6gRf?Eovd30qEz6&<)q7=tRqRD>pH|8S;vR6d}ID{%($1IT~C0 zhAtz$_ShH57E5?umd>iwNJ=NW{BV53eY3x#2qop{Q_NSHN5_}|U#XKsRW6@oxeRH} z8Wn{OhSU-?IL(PsaVGJ*K74AM{g&q4!I0Jyc-NRzEr^kZ-k%Lz#H&O+p?8vUZI{C= zu);3L_QG;Oy2%8%$aVU}8u?Du5-~vhJUfkNx0|j7aoGIaQfAg?)-KI!Q(B+z<>0Kp zDogSTzIHdOswyL4%mfiU`JAGol5Ek-II^m$QXIR5#lC_Y`Sd^U!*Q-Ku6Lp<9RzvijuzrY22JLE3h{B@5fOWbzXLhi2zz8Mm` zdT0v~$+Z!QJG{QsPw&llM8pF(vF9qAqWeIJM}2(xuGN z5A}k)ONb}Yv&tbhm+j=MKF?zJ`~JDW=vxAwRz@4@54Vu#E^&Q>g}{wU&%uo_R# z@i33Io|kKgi=r02;D}FbDAi6vZKZWTh@H7DP8u5y9W6xx=eq@hR7ML`Ah$lar7J4n zK+O2>Yl8)B5voeXhh7jqz8X~?JU$P<;p(tUu3v_2-$AX+L=bMDTHhi%^*IF|PTC4K zA8k%mZAI_Pd7o^Z-r~5cw=jG^_c=BY58Do+<5yd;CQsy${Zd25?%|irvpiXiTIx@H zw-$ZjX2L>#i#WPZo28Lsf~H#{&T6%FCrib(skEj_vuN}^k)P>>zJe0aSZ!USJyBDN z^p+Ruwdr<-i*Lsx(&HOjgQHYjNujGJA(0IAf0z7t@hHgZ|83Gch$((w$#1!u-+k3; zKu9G}PDP6hS=IkE|MYVdKEZd`?9P z5RmcDM7A)MF%Cc2%8n+4hOhLVuKtB5<0`d3Lh2#5VirgLrr+&nNs2flDJ(bqjRPh* z9p?*bq!jK%ish9k&skkB5UTU84A>wq6|bD$u}U8qU5Z#6zLX>xs(IWz4scfkZhKOU z|H#9L4MtJ0F{KK^X>9|vwiz}wm2C(w;rSJ-JJ>js9$-%X8}aj z&5s$urNA8u?USk@AGY189kHl_EyvAp$kxPsa>>Ei>!1spqlGjMOK$2c=?SOQX8OqA z2pilarEcn$36{0-sPgjAd$6rQ6${~DP`!)ip|wyiB$VDx(giigkoY9!+gC26xJcGd zfjg4yOrOelyf^DgQ0zbRSG;H72?gGJb+qFT#gG5B4#`{W<5TBe=7;+*-W2S#TH1aX z5xq?QjqZHb;hEF+bIu_GR=lxhaN34L+%?FHf7H9CZtCR<-fo8CuMw4z%Ik|s^D3l2 zjKDvTlECvmH9pJD5MRR|@2T$)D(P{?5#ju7zs$l6k{FEAXYICwm+N5_DhIoas$$Nr z2;u!;?dZ~uZJXdGbYEk zk4^04ZvV=!pFfAlNew!Ztj-!OAzT)y=B!MiBIMR z4wDt^X@7$M#>+v%PN07OOfLXvvY_0ywbP@vwX~)rKgpO;KRMwjc*0|%cpJe8z^2sK%5v;|J60K(8=SmyD^3H=35J~(3M`hZR>-Q*Rfmxc!k&J zpY35JSYGh_;mlP};N{ZJy$h;NEvKSeTygYquuA0@(j1;tyL>oS#&3+g<9P~IO~-Xn z#ZFa=uCH`@cOSpNU3n1$!{)2{g!5UT#Wqve3B?BaO!*Fgi!h$7L^UIP`xZN&?Q&kL z1IY!@LXKK)2=!k9f`=)YEt&!HmClYdM#>s}soI88;5ye;I~ zbKm>J*R%glO6A2v>5>-0ax zkvN99;ry-b{CZMlB5gkSe^(@oh{FN9<~R{~jmUPJipm==H&V)5`n`F%Sjp(G zB)%V}8AlWTt)lZ~l{%S!`s?-Mhm9g#YI|(oluM$b9L-`S_S1q(&V^q~A=op&d!hcf z=nJ2qsaXpw2Ks2QZo7O#Dn+GjI(O=&MEJNwma`MoZfZI=q)9g8A7*z)Q?xzMjr+Z) z+FY*Gn!ui}G_RAa>l+WaK_Q|!Nz-l5FSPxj8r-Vm0%k5K1Y^Nls@}xgzW0Gvk7n|R zfsNzf{G!SpY1j-Vkh3eC5rn-^`|_ z1E~16sIJ=>dA=KgiLP=zkcxLMWzGn`cjP2rQ@MTJvEI6aLVf&7LH?NKF)uBNw_rQf zDi*~Mmut_5;HIjEti@He8@zsq)1>N>9Aws%F+`@!(70jop9 zJZ>$%zN}=c+)ms<505)?DdWo|S45p3z)0gNNh+w$Ag8Ukhbr^EqN*Vyw>qxTh?JRW zOE^s&et=P8oX3mC6g9&FwA%gG9S5=5tvaRs5JQbikD!H9ZCkZ{n{Tg(B}R-S(6-z+ z0qFn!gV8Wyx~g4h&5)1KX8ol}`;co-I9r#^QmULBB4{RsdNt)xbD!tA)%wJy2qVJ4 z*(*zN`Dn>#Z$Z4^6Dw1gr&Cs2bR750Is$A$L+-cfWrdKJ${7$E(cL^=GAaA(hQRKyV9;nSatt!PYKm+T^+}% zaS9#Abj5a(jNlCPv6W;1K1j_HU)i01b!F!kJNbiQIL-X>S8=-jnC8inK{qy)L4I^m z^^(8K*7mUSV3080HO@R|rJQ+)WzUer*rHxrP53L~Vyv*{&tTbIWmnSXcPe&$|A8Dv z0VJ9?&ikWD!JqwE@6MU!yxq&DNPNr(6U*?i0Rg(FeECWVM41X2FJ+j1-vQ)KIOF;( zaisscC>_teJLn0YjctEvL0mf-J;_8{%h3Dq=~ok?l9h7jah(QIM8xq6bnm*OWxm#* z-Paz=#n!X0cgG|>9v4A0PFT;#HM~I0m0_2%34NHtYOqVEERsIu+V6QUpv_Wto1D8h z%HpNGo;|wnVfVHYB)oqHR)D_W0T5Jm@i?WHJW6023w>>Fz}liqWNoNlS$D^VrKF-p z{j=tZ$|qkI#>1nzLaF^2EnRisw+)05tZV&QIkvp-S~twV#^9CnO`Y1C@Mm;id<18< zQ$F~Ls#ozOA(%H_`^Kd+`3zRXw35=ixq2RNRTXLY>2}$LM&jK(^un+74acLu0_bEc zQPL{0u{kB<-sY9bwt5-;+b>Xp&Ij{qlw12c@2qnAe@0dOvh7yAKOw8`Xz?9!hkRFl z{Y_7*()q@m;I5ye6r#u<`Im`7SOFQL`uUCzDL8E6N&iHdip;!ZkaHvdHYdxb;{EUF`_7k`_J^+=fczD&P#vST*OdG@YOasL&$_5we*0jf9R)#a1 z7vA*i4_ZuN>8+xoT}tKMW6wUt-78eYuh}>S8`$KY-Nu?&B13gFg+0{ zY?*xG7mDeJ+ByaKT9+hIT|C`Cm0gbTakAEhY@%+g)RdD0jR0Z9tJ;)td3UUkcfbD^ zk1JqiWz#QOzAovlI_rHl_`r@wV_n-z0O#aMzkvpHko)2n>B+wQL%*moEWa?>V=5^OXo!?VjM9J80=rM9HK8Z$> z=C;xP*cUtJT>>sILt}b6i(3Kv&iO!gbGy@O$p4mBA|c*A)@LQ2ca@E(8@VWYbBgNE zxCVh?=t4W%`bIRND?%l#6JOTX`;Fc|qYnZZnH&{D=qD_K@_}4@?EG}`Khjtu7T%|! z%|zc%%ciGc>dO8jrHH5kp5fX)<%wf})$i*?H*5?$16qjSdr&MM)rD@P09|*lIq_@q z9ttwesWe_6aSG^k;n4nDdYYtnq$;Eh{WsF{ z!hoB9qdr={X!vfCem?3l<^-8Y)J$){Sc(tL%N6VWPfGj9rTGv|;k@c{HiLdPd~|xP zrsL@`Bxz3E;OX7nwvp+bb+4qivqHS$OyW}{4ai1~n(8AQ_P(p*?|DF4Sr=@@}{ zBdIjh-n0i>x-cY>whu5*80EF*#kO+;nmC7W_s}7D-8NDZIx82q-iy96g!Z6bW^?Q8T)TTxoT4Kx)s1^$kw z=+C~8I<$9@e2I;911(~IW?dqwn|22z4!oiTAQ5};pmd_r&L}Dg%6$z?ES0h+s&W)# zxAQjMuyKWBLfI=CZ7UVX-UOJsoU!5aDY5w!P32nL7)M#-HRDk5Rw~|ikI%EdSH^vr z#Kw_5#oHMs;K@FKDR|MTYn-GMtkUdn1+u&=_)LF#on?N z)0IGqbI2+WgG-bT#Q}#!E8GcWdvcQawZit~UN9R^v*CH&3wrgY9r=Xy?`V|p?!BrJ`*FB&ML~;hN#nw z8XUg~aQql;KN+y=0`D+{Y#7QW~eSMNnlUDkD zrEptN4b`8gP4Hep6MXT^6K-FyjBJ~l^W?dC=LnQHz%j3zyP! zhA7TR?#hLW6;{WJl2XlJUu9IU;priM#&KT&o^T0!`Rjsw3`Z3k6$B-G#?Ne|GxJ6 zSt$M*`bDlp52KM+6&MxfWkM6~h}M&az!kfdm3moYWC3ip}Dl+fHkob&l3{t!lF<5q_4D*Scq2?b0_ z#X(V~z6#HDU)p5mxv2!y8XLQ8xG4>F5bV1SiItue0w(e>WoiZ#a3)oechmy79u zELPU%vsP4NdI-P{zedh*u{INOXiCB5)+VCroOmJWxt05`HUfNqm1)_IF%z4JfTYxx?tz?(0|ijwW4=aX%4T?GUT&*oP0u3&e4&wNaGXh+?tKkEuzyEHm&>B(j> zt~OY=@lvb|ro4up1y1tWG@i$~4(&?yqOe=H{5KRN(<{w}xMu5=B*kraHj!Ch0io1D|tt#k5*{J6rxXEChp+3(KM?oce7ZZ;eSuu)sW zi0GWKY1fh_xQ9uC^}OOE(pfLf44R#(cwf|v=6Mm-T!0YuHs+FAn z4PHUoaFuEXM+yO0lxVlYb*3#ku4ihkTQo3omNN`M%7cGDT0>%KKTMTm`7VJpyQ!u! zV^t@?T|Fh#N-}*{+S{3o8(c%f7@(pb%_q(j8-OF*qmpoJLzVfP-Oz*z_U_UuTAd^q zO_$m8O4!1l{nxr3F&ad8*I)Yl$kI_P^GkDXm zw+%~u8XFd6R3{4crh1RogrkK%g`nN|g@}Js6VH+edW)|;3MePB2ee1IY;QZ-|4ksj z=+fn+kS8N$d(=X^{~kg?S0T`DG0c`>19gEJdJr#>jF>D%Q6mOo4sQP%jC|CJdN}o$A<9h5NL%CZu3*rbyIoZ`% zUpbyWq6Ru%NIEuOI$JA^^3Y#$EHXRpEUq`*IZdTx0ltCg#g6v9yUzYUS+9=;thnCx z+3x{`n2@PDCHUo4K3O=N|NjS&E}z7}kqQ>6{i;W?=fG;%6#XC%#IhxMsYT1F@XW&T z&KgQ`60(g)nVw{mK8$EXJ|}nehq0F7)`NXvyQ9O?t`(cy6y1gpoE(={{0-FODh?dY zBq)aqwNk|Zb2Qc=1aLOCMzjo_s&fyX;w?h4N^;gYOWIwKt~e67pN!V%%Nurfy_C3O zZU7!>Zf^LgPJG{N1oTu$eCyF2yrc3a#)DMnprBq5{<_B&Iqt)_A_bxfV_8(=FIm~~t)l}b^BB3oIKkaZ+d*$PEjrjjN5lAW1UQiQVaV+j?J?AvIuWM4CuvG2=_ zWiZB=ncsDj9`E~pf6pHtA5^aUx~}s)j^jMe^By=xah7sLo3`6wy>5{l?I6HJy6||r zp>!iMOhUw?Br@y(RD=8zND|$EY9!;6$@Vle{F_USkzxl_uPyu|8>Rl0jUv<&8;;IW zx}9c5N%1xz%RD=hoP1i$%M_eoS>9JD5}4Oo(cHs4X%-8tT)f51IX&l3e|bR%j^*f5 z>bp$ui-ad}6iE&OMk30tVcZf;%KhY$YTbKN)assPo-fGR_?)MRP7Fr(J%1W+7z|*`c1l4aCk|v@!`d&9< zOL#$1DVzy`{8?~8^&cOy&^(Wm)I8V8*@)-{O-}h8=I}4r+Q#)DGi1ypvDN4 zN_5k#==n#SZy3OFWTi9c?5xB4>6xxnEt$Mn!4>UBO$<<|50w3`Z7?gK4g`g6DL`9- zxC0b1;f`2MT@DYou~lJbY*L{?ehaiwJ0R~S3s69HEjyMK%N-jCiRhc3RwX$%*= zrydubr}*n`qrdr`WCjC6>k$|Etct+cvZQx{q|!o>+V3ORdt*UqAIrbNKCJKxa$zws zWW*y!5sSZP>caewS;u;RcUUkQXHdr^jaY@bGb@|~w-51Vt= z44C64@-+5*29+$^FsBkv{h+wfdZmb0|79IeTwnwXThA6V*oH$CZ|zz7>w}^pnsm7U$qgjtKCFI}`m4BCNGzM9WI;y77& zUD{;d32?~=se4bZ3kU+Eil{4mnP~2PqaP8Ah)B}^kPV6^mKj+>p+M-p+pZnT`4T`9 z@Z(X)m;i*sB&a%dGfm@5sc4SPk_M+RQ=sPcEqQ7iFb8406ts5HO z1Qb)O(Xmz$6$OPlXwvMxR;8;@IHb`D2i{%;P-?N|S)2-+{>*IffJm@=BJofmySNnsbQRj>sj=J8uH6mtXl=k$} zEsY%u52I@TKtcqg#p!_zE$5BelP|d@5VMj5k{nfniP@)>j+cjX)TN<~P13GHLvR%Ao4a*E|HMAlZ8I*w3fDFLTZmC8KsU zp`(^RBozVJTL}U^IFAN*bCf_5i=YHcl*$8*(T5k37au6h&9uW$*2$HXa`~+g)RXVF zY?KFp3cTwjf8XIvllCTJ-H7ao77#G(iHta~Ee9mJ(XVMN%vZiAhF0On8AR5K8*}PG zX1?lQo^fmT)WtvB{!M$2oGt!Zk%?7GM<+c({+qB+8-F^AM5-<$JZ~tFUNA5t{W*-$ zX29(`h~HO6x}3c}*>~=Ov?-Nia27eGx`1(}9jsbUKU&v*lDK+5f+*i31ZfJ6Iz_(# z;8hd-@=^SBDW{?ExpT?&F^ssm!qS--1arH%n}XR)Ny!_9!%Hv$Z?lBk$7g4s-(_Mg zM4IzPhu=&Xw~$-Q&5=J%Zix58|BwD>I4#_3XLu1h!3kR&ABrU5ZT(*s1-*WNlIAid*y`s=P-x? zMn2EEc)?Bc?oP)yp(9UVsxSChoIc7_bcI|r@pnkWyIE?yL;Kcvt0 zFWt)Mvm>+mQWRiD&g{G#VUpq<~Iaj&-gc%7MMC12*^~-aoiykm%-62ahG+P2?HcGNmVAYr< zcD?o4kU7daJbTtM-A%td>X80Rx^oTawb6*H?hbmr*jy)~4RU}9NSm@D?7ztyb5p9lyrnRzv4E1iP}lJL8gjB~ zzpBWeEftDaR3Yu_CT`+)@|jt4bZ>}O!}tr$LwQimW3}?ldSRhReQ%9NpXviJeUe3R zJ_QbchjdFfXejq{;xE;*48SAtC4o~|lu~hw3GqR$8*qM<2;^W-szGR9UL#0_ zfM%6VaqV7DN0gT4O{Jr7k)Dtov#4sy@fX8xqFJBh!+_;m7~hbj|MxpS!$*r!S?T)% z5QSh(HTE?*hMPs49N2TqkuAwLF_E5g04w0hX)Ju;b$Bs~l;rf(>!C;gVORW671@t( zbe^B$2n-F0ShufxgjhGAk`r(QYU^~9jV$2HD-}zUFCO-shhy_@XKx{jTtfjj&ON6N z@c@}tQ&+E&=J6I;Hcv#_5S?aCiWfc%n6vtg#s`ypcc?*G?B9I=C4yB5siQGr!rPI$ z1$|RA-LoMv9mOwEwH1mMWJ|fU&-F(S@qenf;oEAC5m5B``t{hNXhq&tK<`Cvr=2>V zp?Ix%2lmIQKD3vKI}7_1VE3~SyYFZP0O|f=I#7x4L%|~o70$aZxHNa;5bXZNBfVw5 z4}0+Vb~BQdr|ej4bJj6o^+yn47M-78laZd+Ct#VKiAF?yu}HDP_c#Y>AHc8M8i#6& zsY3s4YrpvjFniW+#=7yp&;n@r>U1@l2YxMof4$1+h4=kYWnF}sW8Yo!$XnL+q{05v z#hZ6fNY%%G@z~;ztUKMLZ=bN~>Asiyr4_%t`Xzem>kIF)~h%4cBf9={N7u zVRrMI0%HUsg~I5&5Zx^dQtWX+eoeZsW^O(hV4eWZsm1U<++w1jNL|MLOS<}_HA~pb zGz+Ws(!sJY`0RDXl&R1IoC z9r^wH%l~;-FW3~YQ|%eG1>1M(5#5CPP@}!2rESKr$YS>!Aa^jn;6sBH?)-Y6EZyIu zXTL*OsD}dFh^UJE+aNQA8Jqz_BrSYyDu_A1_J93MxyLNczzngfT(Q2=(AfPClIN*n z=2Mp53Q(sK_n3I#3d^f+MTB#L*FTi(vN4u9y>>1Ps=Mv%jfE^m8xE>dH}yBj6er;> zN_v-&nN;Vh!T6zmK|}F1WgNuQvk&P?~Q*;m$+4`}nqN=C5|YT6ge~ zQAItlOIU5%CDL4lKi#W^1=7X5*xYUb{T;%4uq_F1v*%{QRHY;y+4&_fulimUo7g)Q zgvbK5H}alahI8-NXyu4oR1_*964YIKA@%>H4~hd9&b9v4;AKq%>n`Vk1>#TNPG=z) zzlhmLj2$)elaU~k-G@jCVTqm}V3!&ATH0>%8gzEtNdDxt5>2k5qdE1!n_WEtf+g;+ zC-d8l_6Epp`~5e5!F^YsW{ffrv=@}Ofg%B6AI!n!U#b1NMFN8&+H%GDw_?~HBG#5f z6b@ebb2-0JjR-u+}c=&A5<1o zn~Er~JfkMD)9KRZn?MRQ#B+q=-;+YbfoQYsIsw$Dat08q!{t%d+j{}3&a{M{487$v zfo>Ru4=@>a)Q>h5-Y2V!ghyL)BPU5E?OV#BFK?pCGf!YnQI(W<#CmpG+6Q@LFrNQW z!9@|wxcQM327U!`CpDn6^Q*!}(wn{cI@NRr%~d_epsGmiRP^0{%#qi3v)sd+fvK8M zshMK&7di7ke=-p4mF%dGiHO+%B8o%%CRv|@($xZd&r0xKss}ojbD2Eb_O^_$urZdUB83U|>uGS~X#A~hB64ZNH4g2p z1EU;u_W{UC`tkFM0&%2CRLLQbG;tIt;6ka)T%fhV72(o~b;!p60VeSbgU9abu~Fa4 zQ{sd;?)&Db+Cd>cmL9u!s_D4pFTIt#dd!N;c>g;&!c#CsU-9%_xx6GzfNP^Y>zWi$ zPHz^1568@mHwT7r1h1twh@LqRs9th+i;Dj}i8tq=NVu-{YuC1|yXm!|M%&jFlkF8^ zC*k|^yf6OoQ6Nsg!_{o1asR83M&U&+&R=ZHauJkr1&tMDzts%{@F<jWJ6@l~gAR;WRuz{l(mlc{zv3-t*;Y;}Z{+>Npvt0kfpdB{JuDs^p zY7HNE#cvz6rp)riYgXpc>nyh3HZ`@%*qjTB#BZ5ZZ1?V`{sd?7wRgm#hJir5gV z^1n0K_0i|aI}Pj8-*CnR;u@t^98#fCq9Q#~NB*eQPk9nmK*_XEM9an?-qZZ@+{x2@ z8jJx-4luKT@b#jD!*7ycS@zWF4_+2cP9akKZA{ddpp&$Z>eQA~m?zi!^iN_Q}NFQuwr3Xt=Xqc%{r z9%aN9;;MkQ{a@qd|8JY1B&@{9IGlEKNYOBhow)Y+^sQ)P2Y_HU4yRAZp$)~wK<|3* zr%$uE{E0o($3E4Bz;3axs!eCZI=F|Wf{a6dqHcPPisu+(H>@FxKN6Rs-J2rmaSG%3 zE0W`1E32>w$$>_RdiVfFaKOvuv;DN4PZZBhIXZ&|j#osvSug2deKg9*704^bCdDb7 zUY%xY@u=;8+nj)R4*=`=={V56bV`J~TGx0N>23h7GWq7SIMzV?VscJUQS_&@Oq~B?&D3 zT?`1)0~Dk}q`O?o?V}=qrCedOCeH~}2ztz@plj;a&Y5ZD3OR_agT4mrP2PLTaUag2 z`ihE1p7kyD${&qFterp<+o_j?3d*u=KTw-_>E5aNicyAgc{%s(DNBIP=Y*3dI@1bq z_ngmxEwaw9c}{OpEBUY1?0-)&U=OW^6;rQP+Vj(u=fE)*--lG`mqx+$`3|8{m*fN? zm;F!vwSb%0C}4Vj7>sl2tdG2QYWQ{H8;%{sxM2|E#466bHQ5xiwonAX z<);{m?is%oW!GZc7UPDGwLFYbD<8L2psJLYmetB~T7;#&tSgl}`}0q-RXn}N;_!lk z&dt?dX)m$q855^$zrQtm@sN#a%y*d0sk)_#pr<Wy>jhy-$oi2e*n#k@ijobK*_(ymhp%XBLfw(JxfZx zCiRE3Ej$f1PbtVq3WGX|&sJIU?}`L=0Sp^3bo&weHgqe%?E^YGS{WU4LOoBPmlSLn zgz_vu$g_&vr~mAx0E15rE%EXuc4*Z~C}sc5aHYTaxxqhi*8}SrkDg2=UveNkXIy+) zEIu=&RTS``I@C9rd3DUsdvPTPZEus13wtEFuK7^r)k^RN7W$KW8s?#(?XYjs=oRzr zF(+CzF_@QXUQ&@D8n9^vy?MyIQRIc3|L`t?DE&Ai zlu@TNeyt}FIQ+NZlz^Ad1*Z%1e?WsWW3VHiCw@?}vT7Z8x@u-GX?l4%UPTkeQ-QU^ z#w-Xxk@cnWjK$!J>37$4tRrtm=m~>c#g&ZeaLsk zf5B@zz;pC5@Q_vTbkHo6c=DiG!rqDUx65uEue>$*@n+9PZDY=)aS@!myL)KZ?{G}! z%?21eXG|?O}&({;Yzp6apf)Y{NRwClP#X&JZ=9E@A?XNvd2vQLwq)ssE>43_{?*Hzx zQ%r*JokjNX=50;eKHMxV!_jKx0qZAj&M6tcG*1XX8XY}k9;Lq-5)(TZIBCy!b>}#T zcl12tVO4JCU#EM2Q=R-`5ugQWlgVLHM#|#Q#~q+Kw&e~V{l0rp9rPg5(Fi!gdp-JR zajjD2vZR>;;iE*Ncx=YP2i&{BWssjGr!2E_ph9;dJ^d1f9%Q8ng=UJ$waL}P{$nPl zYu^3hLUX9HcUqGF(dr0^cR~i>`b6T?ell;@S(oTVmQx=+v6gX~_;S)mwy0<#XqmtH zN$uLo3--US;<%yv7a}Ys9Q=5`zF?2o&m@LxROw$2AIiKPfSWWOU)#Cg?qqp{NJ$4D zSkK+T_>t?RDnIWFDRT~#lu-Q6B+99yxg;4^{>J8|&97#?$XoK%-d8h=G zf4~HwQ%A)KJF(+9cPrH}!O}>!sN#2DN77LJ4=8oZVp7kyN@0If&pu5+m!5gny8YM! zcaTt-z2QG<)bgp;q*j%8DQAD5Nr#zcE??d{ZqPeLi@(=v<<}?JXyl%vx<7Ljd8g?g z;JfzH(fJU+f`S;&W(sm!xjrvN=gU7YuCs^s6K?cQjDT<%7>E{j=qF5Z%exwd&QeEe z@`WMW+lHU78LrO{D04D=KSV3qnEzlO-UrCrr|?CGRZ~a~-es#$cMcZ&n6erj>yWbf zTFk#o0zyYOh!176aPEXt+u(RAkG+~*#f0&UYvh&v4mdWZ{Am5n@{HfV&VOE^FIo9l zS74ow{F$uGj;%ZGMH=NQi3y)ygqj)hfQAR0`r2qRp+4Ew-c%{Eg&snvm88>CrP2w@ z=%%uo^5N3^?-EcZhvO)fFWU$f1NUwhYC{&xd^6fu)6cs~nWt`Q)0NQl^di(g3!ge@ zQ;w8dz+CZG5^JC9NIfjQUP`@r?tffW`;?%4a*K8g*}hK|--^%iT!>r5E1c9*yf6Ky zok3*ZtIwQUY1*DXZ*sDm-(bw3`E&d^ZkL|+2^Pe=cL{c@Vz?2M-(*SSGQlS6^^EJs zrHK|e`F60Eo28=aL#4!1^m%TpbNy%`J8GKi3U00}`#JmI-0ql*@P;7s-R)QbxSb^A zJR1GYA{JX|}rAs+# z@7)Hpq04pZ<{k)w|H!Cx+V1_Orw@wej;4NpjeQndT82X|sz+wpX<`4i*a1X)IT5SN ztx%Y9n;Hokz9At(@MeQ`EX#@Fsw+K*{2yaMKRY2mS^>1VcKbenT#ILxHM7+B2s!N# zjw5HxtsD0Pdt91%1G_$PFVfMZ#I9&2h8_BI8{_n;UJDB>KyQuJ4fAs;c_Il^TgAP5 z^DgSvc1q-jd~rP;BpXy2-8G{2LH>N_M5NysYM|n5Wa!76Ob@s1)5dCrte^n0T>mq4 zdjNStpT(qE_oCQSYxYU$@;^X85^5<81D;;!seliscL;O$-$z?7WyAXq!IfuSJDzdV z)_<^^7-PqXr8t@Fs)+)4{d0^hWkB^xX2@9wM5( zy{LkdH5Zr@{$+p{p}z2qpdj#I2SvqaP1!dyEBlS!e6T3h!>f6grCqB+%AxM)VJ>N4 zAIx>o-}akBxI%y)EQxXG_OSGJS&WWu!sF_rOA-==sZ@rtn@QgdUDiWz=GDC&(Qiv) z{8tS@|K=Z5!jo8~I@5dP>xj?owt`P*buy#AFEs1qeNS+D6q3vkNG7qxiVj@VihkID zD)YQeown0_P1WAXOjob*C*b!TZ8~s$&FU$hj5TsWxe0KxGU9kNZKI1z7NDa!-5<21 zFr%;^G~6^9)!V)QnXD+ljyy+J)O>iD;D@&VXBgsGOFTt)8f zPv^=ByXCd^#s|E3DB~pn!^f!*QXKMLt%4pL(hMq2jn&eU8LUlZEBH^9+NGg%CfEC_ zXUTn8y@4%7T;gg*P%+s;&7H^EByEai02skjK4hF$~_EBWrmmVMHhfjdz8a` zoUTdxeq6_Do?(#;IF*9@?^L!UbZ8eMDxMNy1NF4Mg#Fvr2dUD=vK>5;9*w>Q-+YU@ zJ_pvvPbdup?Pg2;_#go#7U+GR?awYI9%aZozt9Xf6oq`YV~BUj2~Mld0W0g3!RHce za^Jehi%9u5M-5zsAx(9u)W|+yW$6)vJNiJZi;Ft#|3)LqlWE4Y8;x3y?k4836P#vQ zilimY$i2?87|{ zaEl!UY3d6P(?VHbFPn6RhY?xA2}>hu|1=#R_hTE)W&F9={~B?<|75eA9OE7#U&XmV z2WbbywY8=44|7ID%8W=a|HMR*L#g#dj1`fQy}ei*9MMmIj_AQw?#m#5E1o(7`OOS4 zo!66liWze}8|H&0s3qeiB>eLtP(p|*W?UGY42#`(+51*_*oYDJ#vnJC(vB=_e_M|L z!Y5%g>P<|5O>}Mf@$}0yC1XAZABGK|MFS;?^|+7O)>%;O%;!@^)+XBAzKyS?e`1W< zng#GeeqcWSA|B;dS(TTK$`Ip6R_h*~F@g_J-op(*Me2eN+^d5h2J4jd^v{~R1%L{f zbH&8PTha;i$~`vNB$yi!v=w_>H-HS7%RfGsc2J)AAbi}M%L(U{9crRlPPK~hcYw~* zK&_^0g(SzNvG*irUg&}R2@;#1?NbJeq;pSli`f=fkLG?8F8`e1NkJwBc-Uq!rhvwS z6WJ-n_bflf$ubpF{;igeA8z4(ueb8#7eUq_wdBG){8JKvi9_^>W{MbWV4UP2d-W(6 zD(YQ@J0YaZb|2ta`uluPXxH1uAS0@4#!7wm9$3?=44nB|YU^|*r-ckBNNnk78&MRR zt^v02&i(qBd))JD`HK!+5Lu+#wZEf?`r>Y)EKaqRCN zCv-e*hgq1r9hAfxi6ex-p;4xTT6K^a_n>y-Bj4x&{vF=Ulu$uV&f_r+!uty5%*hI! z6Ys^?T~Q{64LB=8Jl5a;*y40MZt-_HN3se9iiqEo=o$~)UQFe`_Xu+(?Nt}2)U2T_ zxhB4)ge#Y68gwCu5$bO|Px-(4fB)~pX$vaU{HerheY~%1X*a$~e$#XjY7S~bp4_N} zKaH~Bm|un>=-5XWcT0nI!_y(L9~qSUrLUD-sttaODl#lQRm&wi?E;(x>wUeEpmvAp z1Ky}2b2?Scm3mlNyKtn3j`XsN^qjjl%ZmJ51xf;pN&6`Hh2jwe*uR82b$!kfRbrGenmfz*@=LZF(p*a=Z3yLXupR;1l zg*-X7bg~jQv8O6eOCZaeLXFALIa*=57GEKGmNsV2D#)D>J~Zbk(Bf%g*blBCXmv~Y zO!!T-vH0phrsmJs+6tHv`sBci1Ufc4NYr;Y=s5+Vs-P+J=d%@~KXM~1jN;{%zW~#$ z#bWSfszaSL%h(fF@>PmL`6VI0)$pe$|D*350wDD|;NFdqO6D`ST}Xkw;O2MnLw4Pc zH%`7y<^Qm1zp@H0{a`)}nI-jf4zmURH4=Rh*ijxkbk0)-auaGB-Axd*Z(ysG&abVM zV37%4aKd>Q+8(WtF!6tt{ibe!_SGOLp6!?1e#Sjm8z0vO(UgW759vfcJiKWbLH%}+xN9C-}xfwae;h`YlqimkNa-_hcb8nE<_tL!BaiC3BEQWO_lI`r=5oi@|mpFKEtQ= z8S?n#gxA@dv!V$6_3r@T--uL2HajS$&tb?1@nvherC~LcN*{x`M_kvN^&siA*>All zXyi?jD>rra$ConKWO#^F4KC%p>R{I14S@j|wVfAZ@irihfztQ@}L5Nkewk<^<= ziVBRClz&d^Y3EoBG%pz@N;Cp8zJne_w9-SESW%vfohoplm0)==0HbCeW8;-nJS8O! zHQLdnMfi0XP20EXC0l{>TL_~)gLf^QE@2DgKpUMZ2wGV1*MnZOpetDpVtLsp(F>5Y zb&>7o27ZM;aBrkTj>V;dH2c%32_rA|y;WK$ic%6iU;Rcp+GAu=!krk3FlM30fBN<= zlmhuUom|^Gh#xiap4uh|Y*uNpt@?;e@kql}Gul=M7by2lQ5rQM@4zAb%x}+UW=5ld zbYe>ttAo1+sY&lFs<{A!6HrG?ETdBM<8-%e&gPPYq#$VBd+rBe+siHzmFX4cPP|L% z=vX(HF@6;b6EFQeQ&^lK;$4_DduDSq9N0M@SzyS!#{;!nE!#-s?sT6xHTB~LEG-)G zAOsvP$>v~B(g?HoqLiI+2u34^pPA0}N+Pk^twezGgg8Ij`y0*L_Z^J%m|$)fTaw33 z^b|E|^nNM{|7vsPdFJESE24qo5ST9%+${NZf_@h-mlrMOR@knjk=VuqC+ne3+=U*2 z|Eri8@fm>6`jyn7*MA8CK(bgm^HSAEZXM^q7^y)9k;<{uhmg=E8bH1>A#w2f-))wo z8)t1D1Ke*kwGa2T1TkivhHOg$lj_vPujJg!ay1qwpK2Y@qz=x%_aXvoUStjVtF}Iv zgHGR1x+GY!{z1!ZN#1PWxy)Pdff6zHLYtDTP*4SBCz5-Mni_SDZu24C5_Qleg;$=y zHf|kWS237ZB798Sit!a?vo$;IM#a0`wESeyJ?I5xCE^9ixN7ZyW_@Uv)a;0 zepHk~1}MBR1*Y_=uUpchyok8^@unP%^BF#RTeU&e3N-fsxytl}xNE^d7N*jaY4R=F zBw2^nRg2c@i?U&?%4-aIFb+D*Bg{cJh`*!kTU-NF?kV#2*^R+Kx}MX{BERFbuF-6( ztXP=T4Hvm-sYmGZ@h6YF=Kc#g{>01s-9P~=n>9X?@EPd+RUPSm$T= zumwNv?YOJ9BggsQ&LL&RfRhyztPF4%Yko97LAh8$yXKKK%*qM;8g}TS0`iEmcU~`~ zg(1-{cly^8^-=({y)l>$=9bV_R(MZz!1wJfZ+>#oJ7wxoFK*5H$% zwsjkp-xTfH;vC>#Zs8tgbt$N2Zf%|m1~zT$l>PC8_Og%zWjrV&ft%|8mG z&S$Qtqv9gbu~^D^TrHfi4@||1tf1CPlbW;PZ$VF5w;|cMU4pFe#H3_4t9hmF?5=_& zbsu{}^!3d1t^~+jzus$S8_r)SS+|cI6@7U_bi}JG2A=~**Qw(8Akzew>$)=xW-C8U z+yreWB2~8PeSdZp*fm)}+W(&*$`2KGTNge&y<|(cQpMmS;D&V~wqO-p?_yc6TM zCBOlQVr5_jE~d@mThi;LrK4x%_Ejyd^B%=qR~;_s-*dz_nKMQP9W7)4TIbsT3q zo(5IEP`o+lYQB4nw)%Zy5mM7mJ53BN_12?nz{{+>)5Ku5COsHjwh1UVU~2OoV~`6~ zA|w>G%IvQBO3DQt^$jH*%jmPa(R>Ly##c)HVq0>oM4VRckByUUT;$oz8@yQ3=O%6c z`lbGYaD6i%(TL`*1#H%6u!imOI5?)s1_8+alJYKx5yZ zVdCJ3*nyc%FH5=Z2_s4A24nTI_fo(9SSMy66OvU_5bCHqhH_cLXEpsf>X%|;(7gQ3 zOto;@YFrt))|un=ULT_zE;cl}X)&%NuqkLp?H(Qp6>qaa z_)IxoI3~rth06Z&jtE@PaRO~3myQ*Y?FSzkL}&st$xa*g9;^3B z1<&YS2r1wBb3g`-87Jd)r1JPTI+UsQ@O0g<*nQV4+_amG)}s`}JT;jy<28g|C^0xz z2I;IzonZ$64~-E;>F85428<)4PY}%7h>>JYr%0TqM&YfWssq*pWwtsqWL(_(VIZB=Z>(kEk*zH7hW=gL z#%B>eP<|Oy85I1R;z5470meb~fTDG=%5K*aL!#)fmnE}5me;3T_zR1^e0j>kKh@_) zTRz9$rcybU_Tf_cKi31!j`)WR_W+50>L}Z}It&*w@s(S|>xv0_XmBLt{%(vUr;Yc+ zRwx%EBfR^w=EP~`EUfKfN+Z&gb7$}T(j&V3(nH|~1CUwLK(evdVt^Xu`g-p)b2P_| zXg4|3!-9N++*i7HRi9u-MSQ>I6bBl+wo@jc{hk00&a(aRqX|d0C7UWCdvAL%tj=tF zK$)1_yB^*BS+gR1+!krNPIV|?vig&L?BNVcUZdgZGNWKVRc`Q1T*mvI_rgYNX2suF zF;>|)=vZV5ptv70)*CV$&LxOXxod85)pmd+iJ**^h#7)iKTmpxL7$X26g%X zd>Y>c1nP=Bz(lDfP?AcjnMzt8R?B?TD^qnw{{ApS9|W%96db4{F}g5 zIR8t5R8@&9Zj~Fkscii~%_`xxT;?1z*?6y5kxh~znd%67tZstdVVv{tV~hD<&K=~L zBkzLTM_<4+w=rL3=i(0jWVq;JusK2gW06Vx%HB8P3er5#B#k)rMyazl|Ng$4pJ81A zh@O*`X4>A^GPQ5A`Al$-M$JyFE#)cHe-z(=QD)sS!WGD}6-)-vdv(LI-|7JYy8pGC zb6=f((ia7GVY$U7n>H;+F$aUPi_KtP(r>GcXFsnAT`aUSx`B)qrL62Jm$Gg^BXw0x zByIC&OJXMqyPai&t6M39OhngTQVCPZZe2ZKOso$tvzxtlJeWFk1mWeGnfsYq5-55x zc+K18vHn4CFYhGlzc^wm78GTJWHq2?%ejB+D*)ZJu@r>>*A<~XKDxHzAJ(%z^@zHp&AG|xuNVQ&4`!T1DK zurqO~+Ii=4tA4VM4d-`Y1I2LUpSI4E6f+aR#K(h%zlo0{MEEmep`kS$T@!R5(y?(aRtO~sLKd^}`0uaA9|HO{% zNmXMbsa9eg%x~!oCL5H{Z0Py#YFsIQ;}d8*SIBimi*l|lZ>|SebQ}&@d9GM1n~B)8-lj% z=;?c~~dXL^e>qL-n0M;lA3 z*ST+GrLkt4Inx_Qh-e|bWiji2jAeN~eTk}nYbd=58j?g?t5$I*GW*9yOq7~AG*5%s z*+2|=-_SMPKbfx?JY2y#ZqHS`zqBVT;#+sZ;S<+28f})8mUnKJhP%?5UUdA{ZIIu!)pGvy+Xa@E(j4@fuAdj7 zTcWL?=jh?yhbb+^K?TB+S-p7T-9a%>%_$-MenngTnY)NWI2y0n28o(nPKXh;$5p$n z<&%*#viphSL3qcK)e>zoIT{k}R=L@z|8{*J(&yfhl_`2yj%nMg9u);Q z$Dzlfrt(-D@V^USB*m1A61x2vLnr>aQHuEK3SO#G=#OY zEJs5;1oS%6sKoTKMkWs%t?_zw#W|UEBxN#g-EKf$xS}G;e8FD#R|EoQlN1-&jmw=o z0mGwRt+N0X;n29fXx{ue6wH9-^GLVZ*imW5$;gwIXX{ftj~+_rrxg{pgu*9_CYQKS zaqnyiG@dCMkHfIgwsPv`<{X8%m#<9l_L&{8_ZrS>H1=2V+m_lDrhS_!Ow9muqgRu{ zFPRN*ZK@Z8_2C7$muY#XU3N5iINTK-+D|8}>mOjHWEfjbku*3nR^xoKUD9bL!jLQO zEp1-Qe2hk2ke(`ijDOpZXHCD9^%J=)RT0Tm;i>ZSYI<9?UdeqdI%_4mBs;q|tsY;3 znk)3}zhcYP4)qAo#>WDuh97vtH#9HbQ}bY0Etm?%yRLi1@9Rw=tgNi%M%D)UQZ$G8 z#w#N&EZ^Up^r>f%QaaNQmIQ*OoK{6Kf;Lcmw|4-kC8JqE#^4 z+wB%#I+j7p4fozW8*|xVj5PhsivK_$%gnhfn4T1_uH7Vb@j0gXsyJ4GsO;xuJHckv zxVCHW`t;50^^tm`Hg^U2&`X&UN!dvvEB9s!!g<5VIS8+uG2_^pIV1jNoLyw#jAfL& zzL7p;e%S-&5Z4#1$b59pJ;W5OK28lfFu>%j`yjd;^Ky(W!}$)r^ErMe7i)IV;{#iB z`~LDjDcz?hUfBu2{v+~Vq<=DAQd#MHiOzGLrSq}7Vf+j_Rq&Yim_^qM?=2j7UwSW? z4HF{FZaJ1svoBSvoQ7Wf7#Yp3b$s_2nfshKuva zRc*f~nc2L57JPT_q~AL$K&g;()#U?Y?68P zM${3VSW`;Obaaybk#yMkas~f$nBy4%8HFWZMEfwSUjnO3RYGrT#dAf_$mZtPxD!qn z<7t5*en@r@%DYPOqe{!U?kN}9jrn$F_3;~BS**4ro!2~lc6uAAf|!2RXYkS_ zOlM^Owy%R^AkNw*4yT=nv`>@?a2p&SQOGUSkhJ-*5YY8^xJ*R8qABStsqVS(qF=C$ zP@HlmPhe@**Y_>&OTJ)-2}|j4k)-RDv|b`C97P#l|0-#0%b(p(8{;>}HvANE@5;)? zN#s%H^8?Ni>xKH)#E$l1xB{0)KFcu=d@Z<#MyG3VmSnsheUNJcwx(V?u{h^mK|bcX z%~`_GXam{M@vcv<2X9{w9IKVgte0IKa<`+bQnGGW|*0o z#@6jwPueHfo9BX@Otc(+-p9c4k*!8cR`SE}u&F2B;BjsQ)~Db09r<8?y|O|_{YXyG z-R$5H5xOgU?F%)JNhRW6SWbWF{9<^;hZPPxN1F)NtS71Q zsG;OCCv38d11}SQUI?h~c)U1YNOb8)IHF7v#GSYIKLkXAZqr^Lc>RLm%&ND{vprSv|g?!bBP_N%_lo^zn2U-q-rfu|KSY)m*;LoUiErJ!NX#<9%~> zw_RL##Tt2_PIO?#2;U(!2HLpf&eCQWfxt>z4O?OgY#vSQwo1-^SFSnKO&(k6A(^tF zva!?a2wahEU)#l4xlG#CyWnB_grEB2q*FdyTq^A@Fw|>Qgu-4Ztj=#I^P-+G)%B{1 zOmyb_$GcacXW@M0Ly>09(D|)RfG+3FXLS6{2WwldohREz(V)jP>~=MKUl=@*C;#%F4XB z@=M&bDbJd*Z$k?1F7Q%paq(r(M|a&0i_Xj804i`OxU>gpw|JX6m3BbC2v!XK|I z)f=^U<_nS{Jma0mAB6ZFKuOT;$)xugueR*&H0IB@Wf^`@ib|S`Tp_n~N_N@9rpj-R zn5><+p;Z_DU2G;0{Pf>gy73w2Hgq)W8ozHpn!-Lr1vGagR?onkE6*rp2)20xi?5cq z3vc}e_D*{mWMAuR^=jSu`oQ9=j`p+A-=vu@M(PIffo|4bHxQIoQxfhY!rhVRL0bE1 z^VL=(Ki^EAX-B;CDrslmBBNABE??xBSB4*2(jlN+1Q8$ZYr}F6zZMkYM=Tv3B46iR z`PdBx4}_ooQdbV^)D2gQM=72}`;D_(qPwDG;%jLg#Sf;YHPpy5c&pKr)c-o0_oo=@ zBG76c+Z=l!@tMm6jZ3O~t#`8NDv;*3>ww^x}ARaL&;7&M-r)fMQrkvRY^Z7oI5O}uC z6o>s;00Oi&{3lA%xAoN@QPn4$`~LMxnS?p&Cj$|~6Y(cv__({B+th0tkonf9oqNE4 z1zFBHi|UIyO%6tV8wd0vH?1DL+9g`7gIYWb+0=!z_W&(a>_5#jf{*quFT$@!s8>H zCYn@WJUl$ww{P<{Ha32BCzy2?JEdpd&Aoa@xOIP5XefThN-I8uM>Xgdi7=5`Q2}F9 zzjEbyUO_>fk!v5Xjg5`=-Me>}PJM0;R#+2YyV5mb4kO>~E^;vWu2555o$NF-GsBF_ z?NaeBum zVo>aDVlcYw`pQ5B{$)vD=5AV&_!@;ZnYiWwk>;B=|oS?zUD;m^2x4#FW?}Y68?1KtZO(C zuP;L;QS)Fdlw3uCx6SPD@2R+u`}dM!%1_EUfc zw}Q)T)cU+1EG!sC)xWB$np@McqkAC%Np&wK_Ir-GB8jf1?P(ee%#rz`I302mW7OSz_Wfxq} zu|PHEsL<-v?77`sUZ+GW)*Zv%W|CvnGqdwWda^Pz8+g^iRZZO1rqWf&?H3@Gaquc* zCEsDI3m>SJMzF`xol{3Td(l;0RQ06>s|B5vux+G%(^E+DwYRMLiqRQKDxsE-?eyIV9Lur zEOp;oc47TIlvckP?UA*gZ|Nf~PXBshczKpEO=A*y6{hiH# zQ7F5DTCzT=3iCHZ{~rDQ-t^%^B?KXMK>2!Evpg{l>rlh-`gM+7*}~PZM;YP*tkZ-( z*MVakYG?)L2zY$BUX2XO)WX7|-)3={HXdtt^-g?zyh`v)?*O&HYg`-^ZIYV3KtvPe z-O0E@Q#ZFuIk0$~e$D3M774uj0dp{IEtRFwvH%UZjr!H-f9QQ2NFx>T+wn zc!kz7rL|w&#^xl5S&^GD>(}J_{rffa?J~CmA&5K{iO|WJR){p5+&&7Tjrmfq6!%a3 z4tI8^uSTQbAN%ev+>zaJTEcr2SI{*HcGXxn!- zLoYkx`}gk_Y4lPt4_@txv!rv$C>g#heQa;t#k{)|OlMm&W9Msx9@< zjA{5phLO9Q-zOzqmXR56x*6V)-gK|*!Oxcr%O7hbnK?2NQutc<8fUFIWnPw)$n+X@ ztfd4i_-8i7Nr%d_$}>q=JbwJT+Iem1aj_T4Q8)W_dr3ChGBu652*zOeQzxyaMf3CX zLvExdBX`UE%n`OkQrF$3-R2W7N=Y?OH4p1Ub7C+Cr(%_7vtCojjzubaNrx(D@*gpN z-d#H-aa%jSnjT}=DO6gpg2=4+>^&1K(iz#?(fvx4=}r9rZSej2j1MP?N$bfkgjBAL zps;Fl(Oo}y^hD;*eUB1KaM9DYdHUY^ta_Y!Y4Q3b_29Oiun@ahn@~Tz<#3Fw$qz7{ z|Hc)?Iq})a50QRzwjoSGaAz)oCtOSW2jGNTqXB^**_|n_%c&_Z##WrY?(=X7JO39- zQsW2lt?h2xfAmkg(do!E#_^r%dM;+*DJC6@$b>(*zW;$|;lS__Hh`rm76pg(-#+E5 zwhWn)ohu6sRbLx?&Qup!v2Oa<&Mva9uFm>vCSS6ET$F2xR^9Q^fP$^)7id!he8)5T9Vw_1Pm;51tfMXf78NqfkS@#8`Wl)BCZ8LMhsDo`Qkbw{(avyVBU z$Z@mXxvG-3Sb<~5j$M$HWQnFNrNaV`_~plQj!=jLk)`+&wR;8ztP+I0_}x}eVL^~S z`OI*0p)%$wZ8qdpa~xN~)Sc|qkgVLLwYq6*j)XT=7VZtp`bV>DoEK(n3}!O>16q|c zIU@y@U1^SnRDxRiXjU)V*bGx)ZM*^7MN1Wtm@Y~m+_@HTgku^U6iHg9{8 z*GFz6>zOsyGG8dHLr#uk_`*x0BsETDC1~tgip>f9=0*QxWdP(XEimHus6_jFObw@} zI(jiYTUmT%TD_E-yf??7&^FL4@xaV_ky3BWs3*WJQnA0?j(*DRM=WQ+1CKnkvNSV~ zzgK7*?=`|c@Ycu2XHw32+(g4x-eLH~Y*)^en3$Naer+dQC5pS}2;h z?yKN`&P_$O6_gc=@%sVgV-W_*BhI@<_!Hcep1d^EI9Y(vttmv(s2|jsi=0;;7+qXg z{o%{ly1(l7?c1Fk&;b~H$!Kw8VTAZO6o*@0)T)>b!yJUaFxA| z;M`;40S*~PLR^*8_00Bt6(95W)_>laLF~0KN^)dvD{qk3}!PB6EnHzj*6Y#~b#4$u9KS;I2KR z3OmYo@P9kKJ<{JF+Z$K>He%>qZMWZpp+mUS_w3wTWSRd**j0u_)kSN?!~#hX z36)Oi4n;s(KspBL?v6oJL|W-C5v03gP)fRo7`nTAhflXaYklj;+8sZb_jRE zV38(AKOg%)XF`K}Rw}2|D^@t#Fnce5xQ(B?R_{m9M8=~Mzun*4t5=q5w=(?JWuDz+ zZ+F6fYP>eAzkp`+YKA`{PlVQ2rYj>(D-|3K)sBXSU-MA;R;w;`-1=@?8tc3+;^!S= zHe9ZBG){l?>}XT!_?2nJ7F3`0wHRTc?f(4*$TbPNZCsYl{S23HS7rw&n(`otVh?ft!RG zYYd08i<@euwnv%6)-wJ#pc8;%I1|5k|z#$98M--Qu zz*sw!EGBSxaHF5HMVTn-)%M_d+T`$>&_HJf3XCR7ed=&qS<{&gW4P?-V0y;Y3##8H zI{QQDL6r{(`s=1tc9Rq3Ihm}iHWn9yEhaj_)>mJD>AHznrStBdZ`aZvEDRO=YKE61D!*OqW z*wB5fG8k7`Z-9V@iQ?LwHZXDiOQ!Ri- z_=fB~c`V<-{mrY3SPXWB&0>N<1oPej?)7RYn>E_H=10Yw;)gEy zuIwm$LJJ4%X&X(?MvQYI!Zfj~9(cXADXvR&2QeC>IL7$PA*#jQ@RRH@yJ;LMxK(JY6^ zC7vn?YV-BQPq%z8=b_c}P;3Z)dKH^op3y*xUZ#By2OzV@G40;DeOsptRd2FIo@T9X zbTrP#-(SzECvmE-u5J-L_<=0P@L4ulmZ3~K9Vb`T7Up*U@NnfQuZjFb@ka5m&i;O% z{Z=24SLZrz&6g*~%SV{2$;t+rET~MGJb0D5QdPxYNR$Bu+e~YoWL&zq?Ih#%D=IS> zvGY?zu80~&-wf}ou~y~4kb|19#cq8Cij~Dr$S9%yU0VI%Vw0I*j#!G%_L`z~gKe;5 zpz}MAqhM-U<0Q0@GgQ1>ny{D$Jbfg-VN}>Ca^KcdePa25EIH{*&CG1v^r*jVll5AY zuBh&@5rh1Ay>c`t$wFibDlAh$O;4{v3Lk{{oADM(4@4S;+4MD^#JVwWz~3HC@m8to zq1KGSvvIlfm_WwjL*L-&q6#48Qpc{&%1m8JJFIi_Pn;cuuuosfT+Z01@|`v8lI|4@ zd@aKFw4`lJPB87RtAt6kUky^WZN`3F!6UR~V${VEYwzhdh41et5J?dB29qE1W;S%EW1#jpF%ue4fSnqW$ z=21Rga5KddEu+ztpy?s1FwL`3jg`-`5$Uy;SEqoEZ`v0shA+cmw&3t$kx&V44qn;_ z)vYn0x!!Cyf6xULkix@^~`;J<(a-h4v&7GgY-z|y=g9Pfc zgz^OZq)HQCvvp;Yap>V%-D;w#qAy}pArIHD=y6{PZ1mRe>Q*Fe;~l1hY>ZyjB`EBD zg9!RQ5u3x49Q&>8^2*J#^-SMa?5IebR4LBn(a&hVA<+sDIUK)+G(s+3{&mPJ2K|QW zB2lO`+a;Cp+@IGBpk{)#RFo5C!M^Xd9$lsbRnmO<@~Tcbg^4qkhl+3(ql6jAd%ub%pY`^i3Z@Pg19dpA~_MWY4!B^vy0W(*mtda_bXxBKA~FW!Z$i zg?T1hQH!c3Cc&(Q*@N>YCKjbcb;?6B!DRyZnGy{}+eJ;7J6X8VUn5SA7bk}B%W4%w zwm>8eO>i__7V>v=fCK|DY%QmTZHQ@9w@B{~jC%?hD&K78wJ3)QC%6KVJP{}22e3y- z%IU+PkL~a~y5rYjH)?pEPz%)!lpGZnc8j{)c5TI~77~AbQG&3Ld`AqhQ*yW!cQ8M)P!@(=)|Akoqo!^mBplm~d~l3#=DpxzLI}@=K#HG)_s!VZ4-6*-8Z7eI2A|I#XsfBiTW6V&gP8 zZs_OEDi9ib1{g!&USf}I^A?1>)s5zCB9CS@d)1ko@{S9DeiL9Kcg+kblZpEILePwv zQ92!+koT6|c97U6fgL9G0^?$U7u=(6DlqeDj@j*Y62k>XJ?LmSS>CZtu&L$Pw>An% z{5cFlK3KoFO{;5$geup~{B4mOk*R^<`SDiDLmE6G2&`|X1wQE!4e0!b`IIUkWGuTvm$L^+?urL$@65$~id*5m67-M@hCSLJ6 zH>a+iHsVuMx1MTwh{4-}{V6yl*dk_6IBkB$@tr$_+!W-JQ9tjqv9UE> zLgUV!O`LsJpM*H7s5k^A;`u@MjLM{c>;Oq5TX!pABgE9Z>`tU_lt@yKy?0&#krwQ} zUe0|5@$|ymu$qE`N-gMeZvi}dIlxQpQOJ>xuSCN)ct=6-RQgbnzK<{;9UjO2VJi6; z&VowOp;i03IqJfMX1*_8pK{ZIBK3}c21Dtt4HGHTG3~46gcpR{-%kpm)8xOcnU(Wn zC$z!=^O44z8B9xlF?d0jJm5D3|E(b;Tn@`Ml6Gc3BLNVis@!plgQEcr%s*HdOEK0< z4rHuOcIhDnzH(q@*mas0RTv)+>mpv10YR?A>i26?nPtlX06Z(yP#y8)AadkD2z;~= z+XP_cU&scIeA$LS0SI=z~Ry;g`mn;u%ocR{aYE)SB1rm<=dLCi3ljw3>sP3aE2kB3Te z-b?<7F4?2H7>FofS&baV3T@&$N|05DXScbq*F?vwy)!@wB;`|58oZD$+3P+|D-X|6 z9NKQO83B#Y9~6SqP2e+{I(tytTPjBB5R-Z!;TT-$()>{f_A!N9h~(g4+NCF!KAT!r z+@~cR86J7)v}|Zeq?~)z5#v{Bw)Q$Ri+t0z?aoEnNO#2%(WQz3C{()G%&hEnNnQOW z6C)?(C4u20E+xo6fS}^Dix?k6(Phg@t>GpQ?csM!=2lj44;yW9 zE{6nGk*kFQeY&fzV1owN6Zkv3Qpi>7szbwgKu_8 z2{Sd%r&n_}Wwg+xH+%u|cvoiG>{f2)<-wAVixd8QA%N=;6A%!PX1X4kM^?3;H*5Gd z_edf|q+u@vfD!aRXaledG(JB5TgLVrc_9ugxz4~cgVs?UnjCC^krA5@|4Q#7V!JX- zD<{{`a4_Icwm4sG445~&$%0<>z@|`om5!(a(s@G+I~Nxh>p^zvb52c{cg?X7WzYTJ`+JiRGl4&*qa$~1Rd0us`rm+Dzo7Y=qc{ice17gUv6i0YiqV5yF-JGVA;_nj>uEYR!k zQMTQvXnTQ`mzn&7w@+Au^t!muS5mox#=}bMFQ_AS8@HdF1aXlZ$CLfwy`2ewRAaY3 zqPllSga8OG%$aV?@?fiM65i{@O8S1mPwR@^iO_wfQ=?#ivD zB7ZKwEdmy7o$}LO08pJq+2FKEu^lLIHd1Qhv#V1#N~$YqU>oSGS(jMCi`xqZxl75g z+;m8@3w33d`(-GzobPRn894D3-F1h{tLsM_?1=1Sm#+^yIILF={Eh)bzP=gGqqO$d zRl4I7_{VC7dIj<{i9j+Q3WsIOu(4pOqg4J30#!hWLrG0nY> z2wA`=w1c>atx(bO9CX8H!W+sG@Z zF>}ZgIlfX*z;$^x;Ln$qNv1iwwn3o6)r)6+Sfj*~;C1YP z|48e|_O(aF`33u)%MdNc87Tw(H`943zc@stKVUuLm-x|73IfGPpjk~OWyn@jeZy~rr|Jee0L6NOp1XW({=MkF~ zd0^&%+TGJ5h?>FqIQvL{+%SGeBVz~ChQ-saC4vQ!pC()z>Fy3{DONn^T)yI;l_ z#j+QoV(O{;92y>2AjRjYglS?^H~7rOurvSgsNISKw%xDzeGowmz#rzzS2ROcTRE!%-@WmY!W?F zO3P))S2tkr6%=z@vX6qSFf8o-vX=LuRnMEfP0+UtP;7Tr?fWhF#-S8bWd&NuPAyq~ zMc%_e?gw1YZrr6%=Fg&DDt(q!y?NjBikl3!eknU~0VNyePFSQnyR(0*N5JM$hDko% ztdX&%G96lI!?$j+g@IIO9$5sph6exQ;f1nc*A79xC&TUS*~YEWn7`|hZjn;_v1V!| zmQ&MQIXc!tK$5lQ{S{5;@LHNQp$O$N-EB~K_*Xjn z(O?4Q=IU%?wMyT(p*%Qi(-0T5cwyPu_nSbRy>~_h!HN*XUwG!{4(!d^B>&=SjHy zNFK|WJf^p|8mn>Nr@&8+fODj9Pzuxu#E~F`qD9Uc*OtzX#M08zcKGvJD?z|6tt);T zGi|~aH|J#4@}*Rm@Ae-%Dfn)hjqV*~c3So48wA=!7DKiA*Q`|yO~rC1I>X~>J8IK^ zSSZEbhrc5_;J-t}OB2zY;qAA#06H$^UjO;nQ=}exq^r8Ky{#)HrYGIHvqFsj^C}`q zz1!EzP3yE`6xz#MPBC&QR}{s&!uu(%*TFVJuUFGHLwDiXTd4X(IvOo?p-cP`u8P@= z_Tlb~5fIP3cJjRq&koC0)oOW2nZ)mqnkv3MR3urMQzS7kcx|PzDc#Ms^=;LUD?oZ0 zRgu|gM3PC3V=B@Fv{EnO!k-K(>jsZ$v<-gmgYOPTlt9o9!iq;wGmpBgwARPwMD`Z9 zCn1=JrYF&-sqMuIsR?p)D+!A{h$g~nAks^=qBqm>pZLp*dp}+tuia}gSmGOvk`=l5 z>#qQA%U^}@m4;@p$P#*e&<8~iE=0=18SZR|idG$fuWk6&78^TFHGtF;{mRgFK!I>hR6LFc$g17c7*VlpRIM$>e6StbVI@0f&e9Au7(51$<6c zmF#I<7JGSIMkrzo6Y~oTP0DjVbrHrU^3bA8mWS|KTu@Na2$(A}DnlIvAL!L?i{VLV zC~C4;1K~SWgiNrr>j9|&1IpA?+MqG~k$;(XH-M5hpsDjwk8rg=R`95s!1UZ=Uc#U) z28%s#BT9xrhs{Rocc;9E3bkqp>>k$VA02GmfUjx&;51P%cIjj7T82WE_n?(`_QnC8 zrH%t0QSk=oW+5)^C`dtHZ|o=mjsMzpn#XR_Jw3S<6|F7LROzZ8FPkyB5Ebs|s=(|I zIH{?1qYc=`m-BAjl3G1hWzekwC?2_(qw2M}P7Um@f8}SUoEMj+XpTrJRh)4*&%%Kp z#Y++!qY^#D9|1UG5FeyV|b#oAWom^zFP;80LJRm60k2*D(m z{29)qmz&pZ_$-yIsE#zLuAqp~BoFX2#+IpsQh}i?b8H42*0Y~x#cK-(!SJPy=K8|e z?qsq-GFl?%RktV}Csq}L?eL6ZH~a{!3Wtqp;~xdea|>~vgCgFoB#Zt=Gp*6NrbDG1 zW1KJ`u07aa$kGe^xm9p5uK>B@qGW)`h8Jlqhk@9y~cv*Xe)d13~JquBb%J9=HbPA#v}eY zTJAPbkLbW2m6D8ij%=?XgPkua+lui#j?4H4@JO8$IcS0eLG~gcpHC`+sD1b6 zlM`4{Y?cNJc=WiyB>aN zO@#viJjwRnu+xg?NV4PZjJC_j&jLTOHfmGV{fPHX!bC14oOJL3s)8FZBmC&4Jp|R~ z-8Hnv(NT?V-#Xr{ZrOFfKnr<}IXm)n>i`W?Mdjjtw7e+S5y&`TYcma9IcUXEQd8g% z7$bk`-fOWJkm%)}&l?CGkdjf0U)OAV=(ABDY<4iv>ff`0ifd#xGvBKc9kRC-&bwel zJqZCjdLLA~IzgNySjFl~l{?;>ooozrGonW%&pYwH(|bfeUHQ;d>JMYg8Gf^AUNggy zfj)#b(J04hcJQ?SF=U9}1(jDh5rIC79crz4Ld3p(VM;R@c$xQ}rxP0mqlMSGp1|&B zLm>H=epee?0B0n?#}9OYZwq=R%gV}H0$hr{HGfx{h^W*&=)8=q>|+pcX4Kb$0u~RM zKxr0AIlWuZBS$BvFOws__-xZP$}&cJdU{yN^#@~UuJ=ecmxn7<24FDKXh~rc*S#i? zkVwlwEL{Y}AQVa#u{uu=O>PZH&l#BU2XB5cjwK)JiU-t!RTWzz01-XXw^+h< z#KHTZt0~&Dos?N(?gEQh4@Ru9NlAJk6tKEWQ~A4Et8H8c++b%_@tm#KD!r1C!$n%Y z&PJdzXI-%``j%`b`v9u8u=NeoLzb3oWEONE0-pNSf=W(>9vtSVJO<85jxb0b*=6Q1 zh{lD5ojFAd0s7szImxR-^FZqIsV71Rkobx4sOE98kL2@|$5!(RZv}a2@+5(s-Xh_; znc2kIH7F6657z35ngAvkI%$%mq+}y}1wusnIV3bxxZ!KWb8ZHPhoyy6@g9U;7!K7W z%F(s<+S(-K@|X^Xg#*HBKr@u+6NDhsi{zqUx1U1ytagKftE@@S}SKc zP&B7-U@@FLtl#GbXfsi~HrtblyuIm40Cx>Fzm_I~_zaYy&`RSU^8oyTtdHG7L+W_2B7G%?b8Ryr0L?W46_e(=>PO~OD^nK(_hb_-P_N* zr)CuS#9|FOi*w=c5`3`NQ4MmULK@@4yhuPlrYVz1>xO^uL-2$dCee) zwtGuvuK2#meV0{u)Tx5Kzz35PtK42gy%q+Yg_ZKGbP|Wq66v-UUV%Sl1TqdLsthOptQdc=z+JX|vzPZ9(Ml-2 zEzB?F5N4|yyXQw#DCV^_F|L`f-c(&GUIyfn&P^?x2fGyuVK(f`ED4C+4#iJVBzzGu z+DYN!n0v8esPT7NyHLXZ2*T4Zg<|oL{(O;qn)K5_zJ>Y_`kBMNzf!HD@o_ySZ!5`4 zzk^A*?nY7z@QW&!hjNsmtnyO}!JL_ z85y*qtUWbqhYg|VX*MGV(`;XAM%UJW297`yTBGF2!e9$9Oe|q0MIx>M2S~golupe{ zmp?Nl#SA&C z&?%o3<9M@oW(qXuQ{%E9ykP6A2dOk3fqrF5ADiUnq|~VilUz}G{Hw&>E6aaOBUXae(X9l)Q4%nCMaI(m zA1+Q}XiY4gcP14=E=w6xa6c;(Up$8%dtaLV=O3}E6I$1VR1Ue^i$W6@b=i^zW z{U2lZ53cTw2Bxi#FYs7ScE9oQk!Z2KAIcsa^qd7|xE!*@RFq^cGPpLb>|Xz~w>9&1 zhJ;Ldt@~EyAchA7NCK_5#{jy39qnB^&7c#^c~IXC#@5pmQx-V|Q~~lDs%xWO?H;WZ zd8UW~TQHcGt6^#t1_mI73E{kd_NcwRV!vOh!Fm4-Re*kECAe$(B6Kow$GR!tp-+8U zZbAYSO4l`#$|&)JCYAK*bsEnqXUq>1ca+7)&HGF#{hWCAZ~0q1-p+FG@x6Dg9`T;(ZNa_8N;Eq|5J~XLm^_9@OEjYNYaVzLf!MeDIU0OR`{WisTHUi9W2SX| zC!@Tbvujy@Chh_IAh1<&g`)ed<9Q|U!XS1lrXYKZtQ*9Vqf;ZQpYlS5G$HCtbi7nz zjv7f*rSBWCOd+)`a6e>QOeRaddNa^v7xf4a8CAqtQc6Q;0d%{A3WYy4y#ROLr` z3(xQk70sr^atgKP7>r^2wAO=~<1BUiP@8r=omwg-2rH=*KwEVI$aAcALn_~Ws<&_a z+8W?^@h~p0^+9YZP3T1nNbjNxsEymcEjG^#BK(8h1pj0&MKPba3{>$?Ka5lL#-RW- z8)Z!Y^e7)d)A&by3{iuho)Ls@@J`!f5I!H&-)bttxNwB@ zmaCRoq;c_APSIR43-%ga#2E>Sgedqx6RG;?t8*2M5pZ~V`y3pC|D!Lgv}28OZdpPs z6b>jMW}ybsfeI&4gPX%mVE>sJ<9%c%#hV)sRtN*WTmMM65K0umWI(G(=d>Wmcd8?m zHy)^}&UsExbt*|W9TE-C8jwxiUB0co+w0%S6{hxpqi_&QOG=lHu-fRSIiG2HZB0!9 z^{Z)e6eHvKiM|43^khti5#Wd{|Dj$QspVnfjAa4K;AU3U7qJM{8BN0N8!!M5)U!c7iKo;<0 zI_TOG)t~}b=-kfR+`r>azjslLWZ0#imTt|^S6o7(DSkp>3N(~nBP>})|F&0K^BChv zv5>0eb*>P?`obk*cfX7^94(`zx?r|$xt|j|nSlUB{?WpaaWytwU|9c$+aV`B*@G*a z;M#~%Q3G{ZQWu56^nFwaX2NGj$m86{MI#&{Hp($6_;}GlNAV~-nz;mdZiuk@B)0zb zgC7ipa~trNXOdb}l&-EWrnanuYg$LPs%44$i5@zc&S{Yi$$$Y>8)rw93VzLj{9FDK zv2Q&@YCp8caX~Adx@3fSkr4!`YFw=$lcoj%a`9nLF(pGWuPg)w zOLnKt=o@!HT-3>}x0}Mm;u_0H)IifFSh92Es73{l6@oA6#=1#k^~(*-%;KHqx5xy6 znX78V?(6R$MlAOkw9RIM#N)DR!->Xpx{-kbt^p2!Qn%A!1?O59lsUg`k^d<$(>F2h zB2E4Ulz9Fb_xxM_;;mRto|vx(WfoV;T%!SY1+AdM=3h}|`&>TI*5p}2)KY;;XGlsj z+v_L@GZloho7(IECde^Us1@ji`#BD~h3ZVJa*aI4utA_V=k`y4Zny7R?Do|m#}WHu>Yh?V$!)cMxl z_`6ci1_O=`CK3-nP_R>BMr$wi`F#V|C$Wj$*y+)NkrxFzc^v9}4yIB4Pt%sWXaxr* z8T7Ud`Dp;1libei6O>Gsh%UtMPX-$obeUNR@S@^W6;F+1){<_sNI%O_h#KlLIj&M z=Y!34)KC0TQXXeG8)1)6SbwEiO;LGRqW2^np8)>t@)BEFjelLKsIFCH#~9@AumrE` zCzzESd7N-{F1RhwASr+@&iE?t3-gi3Bz83No%wQ{}En#wTij&AsM6&7AKaB zKLAii5ymr2V*d21+9L)$f>^JNCLB8JPPVLQFhK!Wg883r1;9Ro2bXPanSTzeNi-zi zAD+Ix6R0u`Ew+{I>RK1si7R)Rv@|7f`a6riuCOTiQ7k^ce2+YkDxa!ko@z#BMuw)l zJK5B_x>5aC9SZZK7tD-|GP-+%W>6D>Z7%0mQ*p`V2;d|pSe;{vgu@Q=0i0~?Fu)w7 zb!hKu4|Yfzdg<8dBGw&7R0~jfkjNCCDl@hl@~`7ix=cSrM$M>jaG?qv3DEJ8 zEPoGMFf$PdekHd)<0bX)Q?3sVqU;Nbvh?W4gr+ zixk(;6MnZi)Dz?^Kjx8MeK;9X!U7QefUT@mW0ixP+vn$s;u)|L`q8KB$OJMsJh#@H z26%+w;?s(g0`v>Jg}yd2poJEw7WO|M4Wu!Vig779qPDi(Vgk{2plS#R1N@0|uvd|9 zm?#Yt17PvG50{A|R=@mhWbpeV;wS*)?XCIf^t9%aU%!e=rl;NC3hS?RHUH{9CXi*U z;fB~FmeuSsqt|ESM)>VdN&4?tOpD)?eSKJvazjZ?g~N%?c0{X?#6e6ea9pcVxIQ@% z=A!Z{6s(2ykE(7U3aEsN?~$VUm-8rH`*DqJi@px#!CI5```F5a3N!mgCZZw|EUu}$ zs>bIr1VkAUNHHaOU1w+;$R7-2qMqox_5cAQ?(u+IF7*9z?WZGub3tnA<^}09=>^k? zuv_<9bpwMm(x*^um{4n`vj2J8NoxjR!U-#qm25}G0zT4m^g%fjR%2cigBHIrEeQ=4 z32vcDV8OipSkqHy^9Y#lJ&!YnMG)0m{YBI2&&fd#Rp@{#x~@)b`a4_5@ry{HnFnPO zDFgCS_;qKM1bKGj&_Z8_G`-WZX+Wb_^F-es3F<;cz$jS6_7lStutT?ViFb2|WdseZ z$E9=W>M`6s(q&V5AF zEB+M|U}McYyHcf(6Cut;yqoGG<`5E?;IqAs)~6ii#q{`fh_d3Dp}JoN-b?cQ%r@TV zfUw^*x)x8F)#IHuZTKbWq#6quq%7fggx4gdAtsiwSa}%Iuk`T{{_ypqY!F-QYuy3g zVlG>PV9(wuXzC8eth+`~A&R<(+>UMYk|_kgx0c;MT*yK&Tu6e;RW6Z5M5;wTml zeVG5?@Fx|lcBPN_hE&=TF15`i^+RYm((X19_o9McEr#k1`1(nS6qVU^VGl!cj{X6; z(1CUp58h;-iAJriaMCC0*lRQLuGyLHz7+!>e`cLi1m%rxGoS4paH6rMWNy)s_qA6t zr}tt5$gX;b)I!Bhr@AfSg#C>?>Q9KyALZO9kof8E-`v>T{7Sl1d4tJfO|MeqmRAa} z;Up;3v$eJQ4|n0OynyBoNp)2Hg*obmvf*Z`>t;>>yQ9w>qWDIPMgODx4r*I@ngz7KGl zk#E3~$}4u=U#WB@TbA8tV;}@YSv#N~PEYP9dxK);^vvy_IcE8^bZ>yp`<@%uSFI~A zp!b}P`d4s*sAh%5{M5NlWIhF*=B-&@2q1JWInu@W$dtFXYx0j_1lDUv&hL=M;OKEX zoL*!{L$h(~(r9uUTGO@qtkz@2{Bt2B1qZOW#g<9+kh`cgON|a`DNL?c*{@$pD_+(l z)D(QJ9P@Gq?w>DGzaLRNOmW^aAzqyUaE^nGVT$wxDPd@ma3<7+>(z&v!qEo3Ru<2} za0AzzjF0E;1I{V}-o9I_H;XiKtjFd#uHr)V$pHemdm2rgnH2PajECw{|77JS$;#}K zU&K6PikMupPb1{;F1Y=d53yToVp^&jvLb8@tI_+25s55e8u~g z2R-D+-evUvJWzqQ@+=MvC{u5-FyyUHuDB_mmmKg7O4k<`^*>dVMd-HnEPG;EP2Z2i zB9hH)s0LjiYsx$ET(gf07spU}j;|xH0b=PB zyOP2L#S*>0?exN{XZ?LC+!ojk^zk=^0r`Ie!j|`{q6!^MBZ`rf)2(!CkixjL68au~ zf1w^ufe4j=t`x2h+1#q#lHAh*6kwlr1oJ=||Cok!c%j3?kL^uB7 zwi7=E7Ww6SUP$JYpFa_Z8VuNsBy3SJTrr8%b?`}y`>Hdp zNk+p;EMu@p%jTfOgH7ni;jX-u7{DdAxzm{Du|Ex!J#U!!0rR4Hgxr_;4SzyaKH2_0 zS~|@dTZ%i;LcKns5m#1EOQkGeVaRxKfqi?Y@B4j~y?%M~Qn%4OmCjH^YWVlP9nl|7 zbYtMo3hX7yq}aPVZmUP}Yr!5rKc1Vd5b-z3n_@yD!bnd{r*WG>+kP?Q44G$iF4iFd z7B^cUzpCly{|eNCsYe4`6iS|t>x&dCU~uhn@5qlYm-qm_!*1lGb<57Mag_^bdW=V+^PYwt{f2N1Ww4np-46V)aU1j}94#|@4(x{7b3EU?C+_(bdl^syJ$E%yh)Wz> zwQJfxoisGtnf)-HR320jX4h zrLSt6Kh6Y_q!&pCn0~DZ^8c;X?jy`{kD#|O&)mgNtOS@4ZG2>VM4{wnurq2n;5s{L zme%M@XuRhIZQbha3<`)aJR(_h_}57TXRn^1R(344XsqRE6T)*9*kzFM&;{FPdw^zh zn17@9IRSc+G%2p;ioG&u{P@B>UZe;m{+EcOg2~1%zA7~7d`F9*yBrn>qSl0G%{SZD z{1eNb?Bz!!xlT?M-ARl)LI-g2F_(J~Q(oq9+kIMDfO<946`*)lwcRj)bf@^Ty7mVo zcM0sw{~ZZhKqLApe_T_+Nd$6_HttR`f(0X*00bU-R=0^?GNPoZTAKaVZY{}TR}8ri zXM<6ML2)O#M~`Q673p|4L=cS*G?lQ3x`BjUT@T9UmjWqU>qgYnx}pc)!;-z5pGE_O z3NFfZ=PfWIo{IY$8THnTK1u0w6rtdQB2dCDC9HpHS?>XheHVEI{C-vWPVLM~Gh6$x z`{Ap=nhGo?Fak0>gOd!;uZBVT-?;zl!xLl&w(ylzj+aT}n9v83Q2_8So`>7YE`MmN z2^xx`EM%}aH&xV1Z^*Z)_j+`Uk^ayuJ386_5KmlcvXE-im~!r7$VqfCkZ^gtxjJX? zOU}7ff|&fE_iw-qvcD&MVr%95E=(DJnS98uL-80ao-{PN`hHU#uSfZx$m%~bJ|#Ch zf3?p|mnN);M0Fl(vi`WHh?DUDNjw0BiwhC|C++$3w7G;9!x{E{W%XKbHdqY(s+RH`2%6sR=wNWk?h*_D@lWs)f{aJmbO3ii(*$U$^mi0 zZxku{h0h5Nj#wXm*Ba1$@AssKf5t7v#oX6-mWhmW*0z?Bj3o*I_WJkTJfA6&QdG&p zF`PT_7?GZ2fxs2Q9o}GZt9p62gpk%6Iy@V4;etLFsp;oE>*jvh3(>!Tj+!ox=E$Sv zO9K`<;K=t&2EEIQh@xS4DEd=mtQz=ips_E2orz|#BA?n$>|cfG9lEHSAfk%1YfcsQ z^%s_exxr7fR`m*Lu&2YOR?rA8n*mNFqodrxq&?yZD@N1z(7C@-w3c#U*=tI{^Uy`1 z{@{8c(##^vtLQ7hpkug<;UPnM@%&ONA7>Smba9U_2I71`OU`w>!kVs@1Yz8HB~Ik? zU&zAs=Ii;a$&-Gxx0$$X>AxyI@Rc&e43ZL;Z1RkG$YFh8m{WmyZhdAy1Mm#>czT*@ zq>d;Wl_*`yZUNw69a~hwCpt|g)&2#HYKW$g3t0t}q%1uLctD4?b9#fA1LNVBk9W*K zfw%4s8~wtOif!x5JSMkX%feJz&z;r8-+;P0Svje0c@>E+QKjm3ypWtc)m<`Hg1sji zfZ`a;+HiM6U|1cVx^C+>)Q^%220%e@N*SvH*6}PmV3kHH69L< zve(`?SU*(fn9wDPiSlxY(2kx=$^F!4)1H}|V7&cQ(fHDhbDD$5K{~O1gIS`4&Z|$C zYMn=&3>}qu=k9CcjBA@9M+v}x#-;~e>*eZ+M~w8^y({WREmt_~3`lm2=&G;Dyv7yT zLg0pt0MrnWIxh9;Q^22|yr&ihouB{89~a)J_w)cx^Zi-@rrv`X69P!^oSwe64Bc&? z`udmWJZSd#6?;d>I7ssNU*vgIFZ2};Pe2NK6iDx`BZFre>% z5W0AyH5I`>ZV@4L#S{0^FHL{jW|*etXesE)-M%dEC5HGXU6OhLVtrrgdcwUk9*JLr z6sQlB2Y}S-kRk;Bi*6~DYz)4<_Uk~302|zj1^3i%LiPr1frTF3MnO9`S?4E>RuJC&g0k zY`ww^L}jyC-#Ood3h4O{!1j0$`zXXInx~$_`|hj-NQy%fs9X`aqa_B6|FFy&(%NAa zQ?U}=q-OtzbBXx;^_#OiK?F-%xq1qLujI7}XZH6hjtXtDmlt@#65&q}2nZsyQp0O; zmFY$*5+L50&JHM|!k}2=8>HZeLb6JN?s3DCuLVx{XliP%c+6nMQ{`}% zVnvSgQ48X3N@^8SL{~1{W$QfjHoFcuhscgK0QeDhgx@sS8{O^MBlNUC7tHI>0CtEG zyvA;W^amWO)r0SZfW2%pA>2TxB&QI6QYcF6p8_k77n1LlV}xgO8@L#miqQ1=OUunk22U3<~gc0cK=k>TclUEbKF-@Lw9sPlzx;B`&h>n4xmbzBg88 z-ovd_YC?g!(J#QS_CGBDW%~%oxVp^M_kaTa)_i_20s?A|mPiWI9!vCi32>B4js{dw zEF^3kjsoLx&YG|j%R?;959vz*v@(^NX-DzE>B-n(lsVsWj4uIY647B{Du9dp)1E#1 z2V&ii(Lbq13LuEF*sWeGei*tGUz3${3%?J%i{41@pYq$&Yf1kH3I&`4OB^?{Zaow7 zAv}a(M+2j*yDxD!Z2g4uL=X=DzEN}OQ4ZML2VMQVTHnDwNBPgYSr= zrOAMTLfr3yf_#ujxMbjeRTPM`HQZY84JRv?jK*OeMe2XizT5we_GOO)^&)i(ddf>k zaUC(FDnAk^_qc{lni+M4;pwmMgryYNX7WYjk;~P_;tQ7CMH0=akS6c8qv>a*WCht6 zrRpXeIC0nl`w~46D62|T=Ha?H*I~QEF>sRn`BHM#4eU|Rqw^uouXrBL5;XSiYj*tc zlZGrY#|u;YMpmdbEIm_*-UWYb;@_*A*ycO0<@?!quuDTHhy?y%Q7{)LIR&$U758&6 zm>V{NDR;wJ8V~u%Hj!UuK2n5i?Yp#Ub(n{Ubl-8?a0R&6A_O(Tm9}9WS}Ur_A?Yl3 ztA^~oH|F)wu67Ik`|?aJ)>3St;YxXqP)={NJm|rtjov7!88Htfqtf^p=B|gh_n%$s zymRsX_y@Ca(3=YvQ7&IcC+0uB$bRaEDg2>}$W=4HRAH}aWNKnw-AGL3P@N!BvaOQ4 zpimw4%pHUBC>S5j((31Q|31YUC+w&ECr<@wsA(_9_n`Wrq29pptq9h$ZvtNFwt#z& zU{31ubM8LMVX^@yS-vqSJ8fLUq@|tt*MiN@adPGd?ME?V0!PknA1i6`S2>; z%OQE5aZ}@$Wi9wn)MqZsHjWcKFi8}GfLK^JoHrl-8QbrRSEw~R`S?zbT`#|weRX`S z#m;afOL(bKN=CXeyEv#Wk3QaCqID|68RPG)@V}`tq0o>&W_A8%>iOr*{J~XrF|mqV zYk9-kvb7{KOtwQcvN$U*L-@HrN$B_gosiS{EtDjdCkBsC>f2P6GDPVUOz_!g@v}^2 z`AH<}fnrj5LuJlPQ8Ddh67&%q{|<@M2V6&21dT41^fO0Ryv?=!@r~+v3yDm!XL437 zXOJJ^y>DJgK7YRM^g<&L<;&wk%7Bm8@H6O5aRWazX|dOq`bK5-nDOEZl(Od)e~uYA zJJvgxfRm~w3~M6&Gk4iaZk~%hfed@HWok1uGK6+^AgOuJ9m9eD7^OJz2bKH%DzP8# zyHGPUiB#7z1utlROEN7^QXHNyI+^2~#GHl_%~$XHh$<RXLW4}p=-AO-N0!gr}L>u{e*!dcKWefu_?7AMo&Qs75x?rSj&!@WKI8C z4s(IanpeijCH+jfZAq?%26}B!U{Cd<)XXiGlvwX?-E^Ri{ePwXk{lhAx2d*#cpPk< z8b*@p-6J2f({#F=83vpmAj|7y#tRiStP}YJG1fG`*qKbOhe_?(oA}QM$V1lbW;*Fd ziWP^1D9>1|Mg5C;aIXEe68pasuyE6PX3vwEm7{m~iPV7!eTD5Swi2wra^Ainfq&*9 zh|j3<#rzn`$CAqs7B4i?DB`8XGQ|+PZZXs+`8@SwUU6}hE^g*vapc|bKLSNun1%z8 zi;B=n&ELr|^{eW)%IkmL*5jMOs2Zc%NYBz}G+zmxf63{nL2Nb`c;*3!1vh#XwdO{R zRMZ1CPjx1FS%>*MT3m8kyjJ~To&9#xP%uXYWAU@U!vkK`Cj!`MyUs4ljKA+?efU?t znRK$fNvY-CW+&^K)ZF;*e>+{_Y_AYA&2vyS%*@!W>|ShNxV|dBSKL?F1x_1Da}7iI zhn|$LoM+$bhIsF&fK|ia2eLlOj>@3PQ;njaS{9D;O_!17w)nfGEB*@vTWO^9n&8{$?E5pDRx;cG~yk{gecoju=*X*6gEY9gT zE?@6PEnkC*M}$K<)zxB{4YN#QDX5@55_CFicXUu|E;%V<%GdI4qyD37rXTs8fR}+k`4Bv|tVcJm0l$#9Xa}ep7lveMZV_FHVg;#z5&fGn_)U8Z#MsB}W z82ZmTUE3+pd~3Z?DUVcs)`Bb*=x1ddFE2-I9obB&Z|2#XjIA!B*1!%fmT>=lb?@Ws z{PICwv{w-aN|elM^0>%H7ua`CF_!hvcqczbTX(D3|F^E3N7QtX`Nt)F-M z`_xan{fu#;%8I@+;O|yJSBd#pQ)ahP;j+>QNhjOZ%*@%i@77=AIkye%j>T;j$JC5? zv;#o&2FDWj=8P8f?8Zvcqg8{xq>r(O4z;H@zTA-{=!4BaV6#~!UTLxK>>j3af2fpb z2(Uhlw!A7A8ZsYhstwKm=z1?C`6Ck>Mn~S7twms%xj*wT#~6(aDqr_%Z2m4s|DS$n zj8J2m{gPO_Mn+?99 zT=6?-#S1^Qu8>!3Mh)6*>ntt){iIhL!>f{Fn!;WlKa^^>?-p}x&3KbLdywPLt1hfO zzKuNxw6B_?PRB@c-cK5z(@S&mP!Fud<)Z{vd(~gN=02|T-u-+6tWas)#_>v$$If31 zt@oq*v)zQGui{enlgd8k*#doLDW`MYm)i8`cgp1@n|CtK*Q@%l@DiNwoxy3t*`e}T z-#c1C_q8q|XC3tQL>$MtkTC!vHN3LfwgDT5rP~i6QU|K~I|KcW&}gW4s)Nh>_^S9C zIoBFh4kj{Xiy8T6td{s~;Dq7Xdz3Z_`XTc@d$*kR*kRujS}NRC958?W$H;tpQ#5D3 zE235s)SW(3|N6R&T6Y)e$Z z)3GX@gF&2i665^os?>?17w@aHL5{lROO4#AZ#q6k9KUE)RtDAN>X@8}ieUXfuFYBe z3V;{s7ucG-{z{kN_5U>TW08z25uMWRtt)K{D@ciugbu-+Qye7b0PvM;;$K8j}Vf z8vunN?X8ZqXbzc`Tno zAs`!yD01<#cx?o$lP>bqN$!MeTAncAGs-JU@z}6A%~j5hFvZE6Hk5#7Lp;=s6m@0=0x;KlbPt;m*8@$!0sQ) zsKQU^)21Gy$v2Jok+YgwS}Io4EkVtuwX{{y95FLlN&(r7VCE%m#Z4n z;@|UdzjzE#&dm+&c{qBeBj8+joJE-B0}#l@o6brIiiCxuP;msER?4S~(1E9WvA$F! zFUCBdsz+_wNSe;IN%0h4_@_-^usFGAaLv(6?>J?&v?1o18oKI%%?>Ny?$~$tuBGQD z3o~db(ee4r`ult^O>vxbawjD^3D8(u*#82IMvg)!-uqX!68Xg2_g~`Yk0EtGQpr|} z$+0AKTf8Su$ReN4%zu3IigNchj@^VG6K?XO?_2(!N>O>?d#8%M`2NQ_Eki(~w zfAO>O-fnAOfxpwa_@$x6iCj@Vyh`HZ4W;?9^}Xh=n|kAFWY zOeXNo^~H6k2&eTq6NmY!@++jCxyIRBDBQ^OopVR$mxUu)6^~Ht6R@1R4*Na5XJX{p zSxwRl4(lklIG~1o10e0CVafEE4A^R!9ba@(A$@Z#(>_F?EYMGfT$wU zQ2FhzVM$k88#9tGAT+cTAXPQ7Oj5gV{r_LT+=kC;$?UIfh;nZUk<^8ynrlyAs2)~W zU9xw$r}|y`cXUxc;4#ocH8n%SmuM%qA^(;^d1{(ZiR79CAzw+-x427wogAixwd$&< zOkxDm+3^wTE0^m2G3gP7iaZ44c;^$2BO^xnQqU@w6@EpP%*1(JMno@3*h(%C4iAI|FA6|E5EmtrgyRkuB&s6PJg(^<5oU5MJW= zf#OxC7Z)sB$l9>=d=o>Aoo~IYGxF5anMAEVdwn^4NGvm#y9h(itwSD)Mv}}izYDhr z_X|hXMI0~lk-?d)C{?nTYRq6Z50A3wl|Sw`i9oUSWbgZDy@j%(K~cYm9T1;mt_-fK zQsNICg&EumaqZOi+quQU{rYJz~w*?ddZ6|#XEKDc6o=B)glN#{EJ&^CB~ z+%P>U%E>WbDwZoSF@?epYzW)P2~3Ue+VR`P&%HVVjQMEB{c-X-350Jf*riH-EoC6% zqP1$8DZi9OKnfVfYRV8C;Su~2Q0aev@jHJO~EO# zjcy2pkfLcz9+Q`(q1q`?t^9GaY%!ylc{|2DYB zZcDot!q8K&!ArQoKVr+oOj-?+89B(riNb(oQzqH$+Hy^GrwMm8eDkDN zVmG{Jx%ojd#Mpvm)Y#0z0eNTlbKPH74t3-C-!RShhya_|@h~2d2hCay{wvx{|0f1o z6pHANaM@6iPeW8rm)(J-7acCEW$Qj1I^p&ifi`g#PsjOoTL=Ay^$z1Y+9SSaHNS;9 z8sPteH$=@1j?VKygV!O3Esn12Hym9v7Siz14;FuuHC+5q@l(DE{O#Z;I?{Zq3(sg! ziP(l@XIs^!A;a;sr3vAa4&;B{)JhD?#iKk&BgC)fOd9)GMCJ%YyHA;X;|`M&4N}25 zIra3{KHs*08}7s`7-0VpxzQWs9|9O$qFQz1kpJM}G;KWE*?Y=8XMH)IHk!6gMuSQDDusS-$T-ch=796_ zG#f%rEN5Wy`>&$s8mZ@rrU@diUk?>e?C!P<4yi6lq6|YM0fy;5gO=$%EzxtKEd}Y>oiOXE&;;I zTcoxy++F>l%}By)ierNwubX=ia-p;`pGCb6XTZ<$={!N9W6@eB1S6SYFrWbfN@@J6 z)=y*q|1ZEX5!TODyU>evXfk{KVwlr0GEZyUhChDdVsnE6dw@4@&Zf#u6?i9=>rflM z))-Xcp?{4u88v0maN?fI-D6zvvjvy)-(*hQruZH~Nmis##NiCzE*jjP%8>Y0Ijx0S zY1Eiwn985aamwbd&fN_n+;`8OpNZv7V2qK?vmWx@H=opY8~^6cC~u?ZSz&^D@dL8s z7WsIPti#}av#cSD8kR3e1{2Nxg*QYd>3Fj0&F~C}r_8YX{+nl=a>?sg;=dDn%&xNl zGZRa1t4>r!T>xmX-PqJ9p7?e%_h396ux)uZWCO%`uVm3b#d!jd4Nun!&AT{u>ytYJ zB1il>fWb^WHHU$-E>DfGIz_g%+d45qjam4*pk0((mv1|ttYv0la*>U#9Qko00Gqa% zd;nMl5B@KRH#(QFsgz^@;RY~gBB-N$WSs{dlZbL+_foCO6y;UzO>e@gb{rob;zXy~ zxv*ohh{=a;^8FHuUVRhmgkRUM&H4N(QF+PpT_a z*e}=350GU%he2$)3Itzpv)Hgtu?O!9pz2H-|Nh!P976-)@VLl}WIVTxQ<}S|N_`H; za7hb1bY#K@{P*GJVpdL5Yb_ins9!huwodKWO=kd`uDBQ$x}zaRr zboUgxLq9@{!ladznv|G zK7dR|!atOm4iV0Ycv1dnkF=IyRw%@MnP|*R61GOjSZ{!EPrYA3c&Es&Egx3Ck3ZKn zm34Pf>{6s-ifRoP;1)sbhXVQkhiS-V1Aaof8vLfcOuk>)n7%}! z7Yr*(NaWqjX?05lQhpy&E(`&F=B4v?0Kh77SOUg=V>w4U3JPzx_M7|q#dG9i1>Xtb zKZ9B53z6X1aYL9Rha^{PABoSeU0aUhBb_GJ7ArMzl`U_Z@5VSm?R`I9QS9D^`Q|zZ z#a=296dfd`#5{p*5z)2=CY(}M_Hp@b$95h9eQ3J^jylt`!;Oq}p7BqVGc%<&N$MDl zzv5cM2I)w|h5T^apupd+K7bCJKIv@hFPha>00jVpHv*yLyyN>kF@yWUbN6A5M^qb; z24f1rIjY!x#2-jo{3bA&F`TpHmiL4<8AVDFVdB$4CD_Hw5=d;MX=rQIPnwbAf zp|dKfZ5|&e@>=TvSpm)Lv|xb`t%RiINc(3OI5`Ym)RXvPL_30&V-OMuW|h5^tCxfX zy}I=+SKdW4`TA|o8yH=S9;Ocf=9>^;uh;jD^3@>`F~7C{bT#~p^z6Ws)qI;% z+(Q#LXXCyk5WTBEvK363t!CF#T5n}QAEE0$g}Lsmwy=*n+}?~`S*bEJx%FOnR?EQ9 z<;AGp)eR!LlEHB5SEc%cDia5RP;uY$AYW{+Zf28j5o76?dE)^N&!aflgv0qeO~udW zj7Cu+_qH?8TKB^>$SzFt{00Kn&T3`l+fHKo(r2HOQM|9E$lusf3=Bcz zjE-Exz&J$zXZ^76Q6VkmOh#Yk@xGq1lt0;MvNWdJ0gAk=GE?F-cMD449B^QHs}a6D zI~tHxw3m~!zYYy5`MC_mv+Faw0H47x<8-FoR`tWrq4I{>WWFz)G}+s&=HqQPRweE! z3iG-SrfJ7Cl^WLiBwowAip(b;CtwVruqSi>y%XeBh$?_HzRJs_ZhSG~lbUw6>15Uq zq}@AFO`~Woyu09Uv|2(D%4CLpgK8r3o6PdWRPML=Fp_$V^To$Amhg5>4e`Q6%kk5H z@lwc_i7$XQSW2J2A?*7OYzg~7-|7}}q#^j+OcV=RbL*2JKe=~GP-ariihy<`Gt^wJdpFQIt%4hQn5{0ro%&t7U z@(#lc8s&ld5BO9{j?Z)00hX1%Z#A_VILEo5;}OWUpAS!&~0PHH!ne)%l%qZ9uIkK_X`<8a&v0vby1U#CFAqDoMp z`GJ9){VJS=&N?^spdy}H(X>}3KCiysv8TT4S#O8jZ_~!MFAG7I^Vv|%>R_5rTV26Z zbEoAE2;O0qU!8zBjN0Hdfa^}quqR6t5Ra@sASnF5xK~>i0;mD@ny#LuG+yCShGqk` zvUaM4f4_BbB@*X3MHpCykiG z_6%+W0!Gq-^_7lfXsY*gBl@D(W~Nwt^EF^vHC-RPsJlbUDH3KI?;xu{%f_ODSnADhI8L&NCezafwSqA*jg34Mq5{&e064sy zI!Ml@>RM6Z?5Fi|#6t25Eb>2g=`G}7p%Nf3HJ3Do$EWZYep93{W{%N#u`t%vKG>1F ztbWe@_ScO^gF*@ZhY)Al$CL!}S1n?1DB-PH92Di3k3J&rV-9`R8|C0jIv1-8jxRBC zX&%H6j|FZzP@8~*O-CaC*x8}&4~3(AN3YA}PFZ}QDY|cbdIy`1C~H{(veZoZ!d~+4 zgFQ0ufy$1=G=0f{#YAC)E6tu~YE^%Fv3~)sgnee6y2V%uFFE86MN@*F4HXYNKk=ih zYX%g34TbRezyh}FBLG?I7k*Xh*RsXSko)w6>yl4Cgx-$ib*ROdIXT_7ir$%^n9S0k zTzdlr(`Fq1u_kw_nFF(MD)KkpQ)|PSh+PT&lR0tnL-z?aR6-oc18_2Y;pzU)uT@!6 z#5k?CyyGD^aAgPx2o^ZcNRR;x#%sT)YE{e|p;*TTec0!zbIKi@5L0O}WG-%rk5KE+n-RIj4uW1Bmn<`+R3Q0=2h z)~i_2cHw&a)ki5lls~i?a`R&BP_PeB81}G7e%MQmfkc*ck6B34lTP4&szdq}u!S6Z z^Cna^rBz|HR{{3J{XmdG8jYV2!f8b|XFOaECiIwmK-(sb8ot-t5RiBF&z{X@nZ~T( z=cX()duaASIWNela7hT=ze(XnG|z+3GyWW&Y-&F5rBbY@Fv_mGp%{~+($7D`%sNmB z_a3~R@~-dOw`o*tqryFF%^qLYvR9P=`t z10w$CmxGEsucP~cN>xhKH$U2>klu{-}Vz zp+LCG@g*xxIowVwslpq#*b5wKp{(YKeV{<>&?P>;XhFlOk0~iB>7PHp0=NFw4h%f8 zU+8kF)3$Ag5iRJ2)suCUkp+teOS8$&VbHCO&nCO^2LAaD^U+}RyNu&X51XnB4U0mq z?f97ZlD&Y;T#8Rrkqd1y^Yj#{qS81dI2gLqTRTF=)0;&iu$vWY>0$gWu>;(6Aq;Qt zPYssVP#QP zm6AxA3$N%=BlToR#VI~q)nSf+qI4za>41H4go|WNgu+~2pr3i?J^$T?Y5)N^VLSvq zj3yPX_SlAHSyv91rbz9<4BRW!@TeoNKY&NQ(Vt|^yt-23<5aKY>cwELkMGRkg@aM)c&3os zdFv$HN>Cs7H(w2{4AXN4ZTbmzrv+rDe0}b75=yvn(7mzl{}H@Q1O){zCOO1oE$xAv9W0n2q=<z!lRYFgHdl6K=%l{Fb+3W|>h8ElojD-`_1H+WzXcqwf3Md989BvUW3!Bj z;&9S#)0L$IoNW`&CmHhQ@E)~WoShy-W2r0%@Nn6SY?;Di=C9@}q-jTyAQesLD)A(i z{WJH`ve;_rWfLn{EBbtALpg8TVy5$Q=FIB2v&_v@OQ&#+Q4#owg7Jdo+LaZwpwaBt z7l$gBvXbQWykNdAE-rO7A8s{iDT@l|XS4$aN-pv2aXrcNr#k*p(iAh7r?Dq>YYuha zN+iu`u~#o&zMLu=URnXloc}ZI=!!yTdmOL3`V6)~Qvi zB;+@&92!9>Z{<|SGbZ28d><^E5k;VM^I3<;fCwlhsNtt`6+I9`uGxVg=~7hL%yMRb zYJkwR5|NLiNxAq2%7mc{f(%$oMg6D_nFB`$E=+hJR@iT{kF~XFmD)6^ojee`zEZv3 zrnJAx&xNCF4(N`7IBM$A(D*o#b3yLfh%Iipy-y+AxBgUFn#}rQbzQ&oeD6{X`?iE*vDxN4i1h%aJ>eyqYhm1sfIuxtTIRwZpA|y@2_ITN(bv= z>PltSCJkz*;BAE2)`8stFDB$HRGOpmXfwSEbM??yBUyZ!vT&JML1PQ9Pev17bj0-jt!yrh>VQ znKD(UD@P7%IVc;SW4IM*a;~^wrdIJH9YX(cTNA=rDoo6ReUy-ioPDi#zRU7EbbEqo z#8Q1sWAVx`Zf)4$JEYpidfwjNNhJxbZpAHGC9`y{@k?gWm&SO)OGy<$A?-bRh}9wW zklVNvJ@bMW4T#kSgs@*E0zHXuSY84KHH6h^p{uX_3yR<{md$mO&N4xFCB}Ivrjp@> z?RZDK)`P39#j7__{x_*?!*85k19vMKhhL|%NZMVV4n+LSOaD`V+OT_vJP-N?OyUkk zsuA3CHgm|3qfX^7LM*Rlai{Of-2!9L94cGgmovBvyA;Cr7Y`L7^Cg1LUofa~hZ0|6 zcY)Vd@2}n@PJ{gya_ze8}YI(o<_0583L^q7C)`+RsdmD?)zTU{J?8twjDkH2J7+GbiCL>W!eHVy)(8Q=fmR z**Y}ex}ap=HBjr1@%YEw>Z524g8!waGaK^@Pg&|Je%?hS( zDfxp*XZg>@a==v1Fu683fVYoG;tXMxu(>2P*Q`GXt}v+zF)8Z@#=-q#Bl>2Kyq1*H ziXGwUS3Lbqy%86gm9G0*C&l0-FPO#jjeggY@pKU~&IK*PdeY!dNEZE)gC)!Jh>#WD z(n~r;iLKS+!>jW|M1HdT0`J4)$-D{W^VV}CEi>S9-sFcjmnYl;JoOp6EW6DGz~z2- z>ON<+P5{HLnV{!DkxHALCvvAbuEWh$2xK)j*Jaf(shbk-?{ZqAwReQ z>bSFkfkE&I$miGHlAiJvx_tRFNes+c9|XWO-h!4*I#fioK%5uboqG zbL9E+=SK5m?NNa=%$AWxMAC($l-sV4E8euV6U zGp`jRoYxkGL_`|39PP$C@&~6Q*3s^d8kkcmtE#Z)m*uXRBs|USfCtEiaxDkayNc4K zy?MeMKf_4x+msv4tc`mRukpeTjFE0QBbTVDxK;`9R(&sb@&le2W8Lz z7j;^iQ!TEw*Y6T3XJnxwLi*Jh(bg1SYbg&(4?fO(J;eVJcnn8ji#f>uF8y$k_P2M? zq>t3}yszT!)!SuO#Sl+BKdZfLE&bYYn!0=HVD7QBX6zVa`KY_V@X~*<;<5CDMIJuJHt>gd`Fs`P2 zM?pgj$S4%;aWt}7dpauy3;PSz7T25v&?0D;QLT&u)$M?|OmvpGGdV*M>zl_M&w$`teO0sHnQbd$jER`V{~AYfdh8 z9S{57Z@x3Js*c>iYbNIuG5X6-GyqKFvN*0?30k5t^5lU5?kkj#^Gb?hv*vc&ToLGQb3 zP-#N0cf5LX^g^A{Fz2yswe`HDp@N*-*rc3fNdwUc-y5q%Y-^WS=Q45Hv&}eX=cVt4 z(pK%aYuZB*6x6AS`UX|7GH~YA;5evm!wYOxJYJ@I-mO^;7Uc`u*y?e7iuVvpUw=cuzkJ27F5+%n07p*pQ zY6|CtTYB4w>S>NIb%#EV&q|ih8nf|$#X7G|N5p9SaTV*zR7PT3n{Xk(MaD$JDd^=a zZD6jyY?|xL*RpxO0tqUnJVT%ow{~GW3~CR@#S{6e8yv|o^%-+RsC%QkUmG1Y$mqhj zRdbWIhuS^a17|nc{q$=si}|o^8*jm4W7P5Z7!y05q_6izLi@)9I3E-&E;<@Lsz!rw z8(atnaHcUebTezg`s#R0nqjpFy#a^OLP{yCF(+-1GtL8M?r}RFiUC_kTG9-a+sqEK z!xLVGAbkrrJQg00&-*krT^pqsyFm1CJ=R3$D9S8YrLCnb2A{9g?_;`Ku%3)2@eI~Q zfCR$$cy&m^qNkYeXXd6rB%~x(Fk&iFhb;mXIR?cC22_@9>K6E}td0T1vkP<8^YIDn zC$sFNP=u*|aLH%w`WkUv!pjHYz%tI^Bl%i-;R|i_fY@NoN`n%q1)9_a<7cp3LLAOOQH|Nn( zK&F&;-%v<#EMRj>!=LMs$9nzO3a;HVxMq5~U$ZzgIJnjP`v9AO2d+mhQhpMrW)bG= zshJnJk{2fJyi6cQXLi{0+{{xU6c^vH?+>%B1zc1HfE9`&A=$}oadrq;lQ|#~4D=R5 zkOkTH)gZ<~LcyJ2HabShjDg;@2X#t5-oCYbYnQza-ncSpFH>PE6g@}1e?$p0k{do) zI1x}kKn&>PH{AC^AXaz}0-qAUVl$0E3=c)-!9;@`k2z1ZFGvZG`|q2){ZF9^gKJ9E zY{?Uwq5N#Kt?;Q(1tOI2s&Qg(bMWt9Y@yz<%YVqIw-wN?45}N{Wglu8D#G6m=yq}% zhWT-!v}39Vg75@frs%Ap%;a)z{Q<@HuuDhF0&t6!$Y7q9J{o`|Dg#vG;H6$4YxJlr z&b?h5v&9)F6*AvW0@0yqd~ct?urw5^G)mJ6u%EQeU#|}Iv(ew%D}Te=$@jaKR;(zU zM2_3ZvLEpHtM`Ufz$bpCaNk^HSSRt2*?r1U&0Z>IkFlr`dn>$-#xUUlG*X z`m!78YZ#1pm5t?cgq zRGlXNwdV)Ua($kXZ*wwJg?Gf!>rzHU?OgStRlomXL@*kAMPDmUpR4%O+k@xIwnU zgIlIYd}78%{Jd-m)ZF7X7{!L|WMaq85m^6SFMiCH-oq5M_( zH4JZXD+1v}TStDd)71hV$DTc5&d4;OzTWLw06K&RV7LY0=9z*4)*k*8 zJW~k(uDRNLmDXD}T;k4|zRao!8Qh*^^s-i}r$4U~qd@F~&^ztj6JNJ*DM&gd{O>K7 z`OVjA0tZG!aI+Faa~-)0@VSSjQC>tVfxohv!8C$u8D}BKg*~>9Wg-}v30$&gnBIhk zn`tJBmQOqH>o*m5lw(I0uv?s4m;2XXMWuPvrn3-ZaL=q`Z8Vc;mXt%dC>P3KU_uwq zP|ApN?r>gTt&|ZMq!aL1AGHH!A$YXW<0QQ(>5Xq8v%oYjov#OOyku1_)oK!rBB(^v z^keq14%B~qz|>z}t&SasxvY zv1&2F)uOoYylR{E(9lq={9=up*>%L{4KH^(6d9-{UwW_f$1R*Jt7c%3ci-Rko@$b% zSv8Ls;xsY!LP!E(-ve)W1Ixa|mdPFDVkpoe79XLlp^z5juqrV?vzpt7Mbcv4?77H! zbop)foL$n`W+71wj0i3|JU%*AOahF(sGrO9Tnfa9L?CNKUd!u8k#cSj-Q3(} zK>XYYD6fEVUhiocREU{$g+J+8g3425pg*(Lo5@D^>|7+n7&7fL4j7!bhC- zgqhU2^rmjura~IJ3u`7R&-olQwd1I}O=rcl4sQcaki7-P4LwHarvCaOr^s|M?D{ zJ+cd$$?AF)UI&bEtlx4SQCgxPI_be3*5yq~CU|yvwAf^+i)UQnj86OqiYiBD##eVg z|9ypJ)9ynXX`-_cvIQA1sg?UVa@@DTaX)MSsNi1;AB$jd9xf{y{Fd((V84D&_T9ZR zudlhhd|3-PvK@8-;`mH6!!Ga^vOP#iW!3_mp_-lf`6hl1${Kw#C66z~-S81&; z$E{kxjR}^`55`28C4CA>4m^DzxyYtzbJvjhyUhF>osRu$HMomQ$|>o2jY5KYH%2cmI@ zdIB{IJSBJn@VU6?&!?D~HS?1PG(l}F;LJmA6MX%{a!WEwR!a|-46n9}p~aR{Vo z=+yR;?DC}^7WYmpIWTQIjEO;r2XR*OrzeNr)4F+{FA~IJFGx_y)$*y>l>S!k*!Gt{ zy#I$uq<{1F?d|F5>A|6f_{w*vug2%-nKCv0ScHfOaoaI1L0{3K$no_>nRTw4mPl1q z)wD&B?Yg&8=BK4k>XBKklQHG&W9-2@u}+8CcC?s(*QgYwdy?D2{?`&@Epn85SPNE1 zkulYdV^*d-V#$P*`ZHt0K5GL$wFD%tT`9|ZsHLUFAax?GTAEP94M#}ZjA&fcQ$XfT z$_Zvm*n0g5rE3M(?8}O3=D=a;E~%~$$_T6f>kWld z<(3;)l%f{xBOc(QqDV7K#Y#(){6t0KQg_#^{U?g=M50PToP2j6xoymJ*Xe zQo0Loa^yW}Go-e&|7NaBNZm0Oneg&|7-rt1@l$8;e~C| zUR)2CZ&zN<5wSo($pT<1=|1XfRuAz5Yf`80BLznxYBEFXSxQN%ImQnGMV(`+eV8(H z$FiHxN`a5J=JF@7*#G3Y}hrP_xa2N3qQSoW|Oi{&;Dwf za3K#M`x$Vbz1U3@YIt~9G3Ay^I5Zigmg|(S zSvw`=RA(ha7Z9_6`@!*XMKP)oW}L@^2&IlU|Fv#wO3^QUIzHU_<@v?dg7vj!(ot=e zQY$ZqG%`5}hV-Z~eJhx5S-G}6Axq5I$d`(Lf_MTw&(V2gb@V0yhROLtmk={KA zsbm_lxLx}eRX8cneChvQ8jcMY*NL_G*y5FXo=34NCnwyV|^sOZ22FgTgRM}e%anjAZrCBTDp+pm(8 ztnHAY4;owxOrol#@-?k$B9ET=!Fkjz^23J@@sEek7t7U-fHVcs1gArI0{mzVS5*p< zSYUExrWJQQ(s|8rmMW*YrTmTWC1z)AQ6N!WtQ}+rLa^?SdKW156>SSJ%}c!O6^!4&=t~LPPz-bDW+2YZDe+Vlw$)_0fO$rg z@@jp>P4WHBodUC)XiuM3ig0*@a9ltKbmiUrDdj!;FrLzc9t0+o;etE)w@ji6>-fVI z{rZ{UIvE;F3*uag>?0Gh2r!tyAU>!u?8dJxf54=2eMTak?WZ=YU#(0pC5XXcYxmHL zVy%n>CjJaCj7}71F0B2g;GW52%DUeiV7uDR`x}sbf(Q@PMp5_TZS4&?i?1A1L0pPC zQV%u<6qui49U1;@1u8Z6{+WX2H86VzlJ<@93}ImNt)j_;qW}{wqf}-6a}nx={ovL$ z-DYy0&zyrAWKY!+Ue<~%-WBMou^A)RpdzJEVk}_lTjtE}dV?*O))@^hfEiJMx};>5 z1`39(f=aR5tvmT5(thEKO=TIg3gpd(Y|sZ_gS;iV$%aV_2iYLaF?01}!*rfIh29l; zzbW+I-*yeNZ;V+!VigSI08NK94+5Y^)?k|p?AV}=;)U+D7jN-;JQPX4Ge2H{hw{A= zA}12S&_R&ksn+=*MwA}B8#>@xNa`_D4TM~$x5zzxRb-}A28BLuF4??@x?6F~Vc&9X zdr_`>)m;_3v1fhTJ05#=ZO)}ETjg%UZd?72>}@cX`#|y89UOp3JieuiwFfVd+`Iwr z&IA>x$k>}8PnB^#rfd}5Gz!?~8!?2R&scJS|BA;X%5+=OXQAQ_oJvyDL#dr7Kp&2S z@>izIBjh|s^Ha>BI;id{uq%XX*~aSMoCAIrl4F;rdiK>i&I|q0+_j;u7-l}G!oi!3 zMd7(uqh^|W&qGp5DkyNU_>7%Qf#b4Y!~-IIP;o_cg(hs5`2zFFG(nMhm6KCa__n>! zWrli{vV~Ai??nbPQG<9lzZIQwdQSXQUJ}$+!4FDs)#eQ3DG<}9@L){O9@U5KX?X+l z)E3S}YsH7eZ;oK$=jV%>j9r`o`)(F#yxSN2K?TO)={X-EOJCa=jCx1bn?$r_l$1R* z3gqht@+c6XX(r0W^+M`O6sy=!(@uk}lwvk+DV<~imcKa+B-y9>qv4vh<xq zDvKlo;KJEMp2}4*izBmcrtwkGD3GpUPKC`xdLkEX49C34Z%2WgF<0D;k}TJ{7vQMS z1W*FZ$_R}DdE8^pWWb0la}=i-L}(KU#61k;3k0_%qR{)OhK`O|1>@jj2C){>Mt2(2vd>-gQ+JvT zu&tx4-sBB8aog+#H2E57D(r9RjRYMPg8iEv74_dZ|NgC@f7lL|zgoRv|F0tA8=ep zXwwU*alVtO8^&d2s{3B>w~m7%B3Cm_$A~X>ds5Jd4KhDTswLJtR%b*51XjUHz^ApU z|8&BEP$i%qC zf+YV{^`E;)6+l{5@yxFB1JL~neA#HIoau;rnYRIfhW^~DV=frNQt&%xW@3c_T`UkW z6rY?bSqzg-8q2_@H*D1FJ+JforN0>JqktWhAVY^FL}zS2XRRIf>utg?eFZ%$C|bbmVS=XxoF?_jXZ3>9S!n7jOMwc{I$^dCtzaU zz@-}-DrawOv~~wE1=Ze|JHHNsn4e5KM&ce&GcYTzeWz!u(2LGBK$$W5Q5quyJyS;i zrKH#QOpZg1zo^Yx_mm9EN{G!3AUy0`*o+f~j4$~--|p#f03+&FJ7;=-o_YBL81xZ- zs%>ius+#E*VuGroz$lH3+qlZQ`^xD0^USwUI_d*V@##XqjU}||{RTdKR)fSmhW?gI z9hfOgdhOApxhSULB)GfHfY|o6;W%P0b4QyYWtYtWNB@*5ihT4Um$pzN&*-YVAJbhh z&+N%&@ZXI;S23YG4D*?lqHHVORFY;m)#Qr>IcHUO{%@k0=1K}n}f9mf3I#>H(4})K@HIl z-ikdGqQC zq1+Gjm10X+Xr{>8)~n#c7q|Q9wjGQ5j-}y;=w2O7>8gsRWs})$iMLN|*r{v-0@vvc zbs(2YS&Ht!q0bFe>kHl4GgS_Uc>bP#L^M;mC%SlSm!^rolYRlVnD@v>-wWb^Hrf#X-H&jHtNA8YfKe{3qa6c+l zkWEBbGcAKShivwl7FbM;u;ED?eyeoT=CxP0O{nYC`PX`io&z4|UPs#Pkw41rDzV=* zf=Nua=QE=7{2#BfR&o@E#;n4va+aHtZlIUCQ2g4=v}{K_Ni|jEfCvbA)ZXJqk%LX+ zT|0k;b5DE_6#eQI$DlODPIUKSI-)J%#nb1Ipb-$2ra$&U;rjEv6eeSH?HUnO5m%c} z>ew5o&Q_ALGgc(M-A!SmvB~M{c)agKnDc2@UP`%pLw`qh#QEUPzEi``n2+|99aR^I zHH~c+4fppkbYNO{i1lHHIpWc{dFFjZ91aA=>YiWc!~awG=kbA2vje3bxXZ+vMj?lJ zQm96_W<_<6Ycsqdd`V$B#f{L_6L<4}dX()!n)hY@m}GREWZjv?uS=rhB973migJ#b zMn*}S)ejmQbueP_MBCrYvOC(`NhMI((3r~m3|qb04G#-#jx5b9t#9s~QObH|Dv1`l zjC~@`MP4GvW|dPNbj_S1%#ZB+m50u5`FTiJ-@wesX_5~Usi!$4Le#!ND5;sv!LIb{ z(@S37Oq7GxH@>RNatPh0YiKw4R;t1Fjio-RBk!A7$$dOtfHbGqCKJX?lU(2wB^k1d zn*o^}qd)c*Yc9`V^F1nk!uVLQ}=uV(8=A+cSVnwI#p-blX3eKm&m*?(0tR^HZM zg?Fp$mee=PQ{}hwb}hWGdVv#*U|8i_z9irrX!!(X-Y5<4os`ZKKgFlb*}R*GywV-}u901+G?^uRYU&GXMD(={tlK@H;ypnw#q2-ZndSfvj%2 z*AGeF@BP~P5c{HPuCJy4Q+Wz$F?A*HE=I47U;5MmqNDMf9ml?b{U_mY4)fzO9z3mzJdD-YcfX~*KXIf?A|!t_=6YDbi^S$$hEx*@|`o; zA%ABrbefmK*Df5Bn5I2td-zk`Wl=Nrrnw0T{xModI9t-5D9Qos&>D0(4$atP{@3Fo ze*Kna-H3kgVg`woudil^)R{V`>OM`&=kpTjw62hK`7*Zc`_hhngYW)+Gx`z-FAHm< zExt)#YqJO^%I4FaNy(2KpVtvT;XP-q;V5SNY}Zk4TfB?Z-N|Md877CqMk~{*{rU-d z#^ItB)yx4GN-Dpj={j&ibUHFp%>x%KcH|v=X!ZwMAWAZxC0ENCUo@YpT>Igy-(5w^ z2RSdxLqR?c7BdosZ(&DM3G|k$OCi}iM{I*GO)kq z>E063!juweE*m9RigfuB;zznbG1scu64Jvqp83L^AL3!y*28Y2ayIsRsxZwV&8KXo z4C|7sv0?9H((pq?iVp^VwfoHz+p6%RrZIiu5~^n6T6Neh0jv)C=JG=EsQKE^6L1VR z{aO!qqz^iVo76xieVIlDb`A%w3NFgjaAiBm=;;4x;b_XIbu&Jh`7b@hm69*!=14-j z=W%i`;*R^#Ejl21f93HKHqcaYolbo%Xm)y9E+XdyQyp?#`}C$d+@(~9LFmyr>O+{F z7ixT{8@=%L4var<#Cq7V-gT70k++iY6}vE7{5!DYw_|)fB2q(7RnS{5qQXvVntpXh zmALPvdMOvph^Rif17yJm?5lsMx_M_(rMS2MT3EifU^3sMt9Eu0zZXZMJvvFZh{`cp zyZfPyb-6Z6&so=2)3hPrxZdton5{4C3T}vhDXs0xF_y|hJ`eaKut!tOkLwTP)zei~ zD*sb#Dk*w_NqKSg^lopZd*}er!E5}44?BESg4Tu(+6igCJ-7ZpUtoG$fa#U7PUm@M z?JpfDB<>^Dhd*v2q4XluXmc&*`e|KP~m+Si!{o z@sYHTc@J7k8zs4dBKUogn*J^!%%UO|5{JDWdF(Ygc$;?j%s~tZ*g$^dxm}}elskJv zR^P9e+-Q#^KD@K|T+XsW=8-=^4Pm5Z=wS5(X?oD2f#G*FSt9!nLIWM-?d`hC!e87{ zSxkEoxU?nmz4aU3PwT{qc$pa1-J54}54fL@8NB_a?Z09gn8d8@FCV5+ahR%HlwiZXlX-(=wK7rqNnXI#NSbjXV3_ zS>?n_vH!i0(OEn+XSeg$c&>@b#8_bV?k`2_5yh+*vxe@?O6wNS#TDok5SP84cXf7P z<63^}`s=9(SgGTzGHnb6GdvEAQ-&nThv7aI)x)Ir$cK1IefrFzAK&VuMP|)+9tS>r zqPruj6%|SD+mz+pvU}{trkz6T|MC|C%Mm~ypT>E11ujc z@N0!**dLAEGIh*R^uL0R&x_RE#6;(zC$@H^7po1lK~qMtnM0Duc=s?5)Off2!kFfs zi(hw$Sja+YUXy+NHPEh-b!W7nZWI+8G5DZbUCTYL7NdrPoJX`pE7e9K#9JA1#Dken zagwJIkr!>i(A=(_a1EU$_$<9L@tPLjewD@anF&}wN)tAs^}V@x^s>;E0QQj`QwkZ_ z^N^KuYZ^Jb^H(_g)^6+BxZAU_#_`=*`*e5AXViMBU}e$q{}J}(@lfyI|Mx1lqN2no zqH;@Fld>~f-4ZG>$}*x-_9e?$b8oVRT%oL~BsL-vTtJ<8k21-gTZXS*P9IP z_4$4On} zpUv#t!Zd@ln^0`wmGrE!ZHn-6{|vt-#%475JK*+;(|5IgjC7Y?$dXuf%hjI@UPNE* zb$;qmC8rNy<Db6jtT*z`N06uWiq5Gr|C%a( zQrrF72M_JzUV6s`5r$b*skd0cp46p)MNbNeat^7bFW)V-?y*1rr?$CF)3_j$`vRHVN7$$x6RmU+KjRB!q++Jl#|2F-ZqkSNwZE|2 zIjxTQOKJiCr4*3ux)$B8F#SpINbh5A3u(a-yaxNDV0^H%5V|a;zSTXPv9)F0joIb* zK~h!R)%StG%#ihTyrc8qSGGQyK9fK_o8Zo$8397vwuc7Uj0wynyN9w>Q1HJ0db+By z`t~a1xVgncwrN{V!FGDjyO7+eaWc z+S})T5gA6GQ84_zwHB6v-vANOP!@O0BU^xZWirU(k^;gn{pKqS)4ECj!09$a>WuJu z=VxUXIFT$FBsxDW%u7IAOJ-ey{nzAvWw{J2J~86_*aM3wt1I$(`?6X~W5Vi(M226z zjM@Rq4Dw~)Vv73Yu-;iLMWDzw>gdX(sNQY+cn`z@n5#xSnelxxV^^#PFm@M&v1HF7 zW4VI-^rp}atZY?7IpMufZ)hd1F~`{Jxm3S&Ir0CGu+cZb>%VtT7hoPX0|*Nxv0Kh_ zIl~)n2p$=o%{^_y-ZO~c+$5I&$`Zb|;7Iw{2>M6Q(R2k&Pl^e_q~@5lZ$fDDzB}tF zn777>*NT=9~hnMu#l`-7t&m>cW>}C9P3RH%LT-bwcEVf z&lx{(5xJ^ihCKYSrLel9zV}*G(oZILT2l7h+TnfXrhAKvb9$Ah=9I{f6Rne-KC?g7 z-Ys#lmag<38yQTfDIqs4$F$_O4KXcR6IyY%L;a@P;Q5Lcm)VmG&M(@j?g{5f|eeGJu3wL+;Z z1FO1L**E=X(S^!f89Rd^GiRSzS7!tG<`3c9o5+#tYxt&S{6X*5U2_G>(sM7%&xL%Y z_6HDOod>_feQU`r!6mS)-?dc}zXu51G>G^IjZk;I&YZ6+N$dmYx-g@J3=q$+wX23t z#sU2zt}y0b5j`1td`id2owdh{KoR}q$vUx&}e zl~8P%%KWwk1e%q7OnYD6GugjWRxN2P^TLfu6K+9X^8`ngE06MIee)vTb0Rc04KCka zs(wv!{3PQ;3`Qg8>qz#fQgOJ=_$kujKMn}B%4FvZS%oV>V);$xlb8+#_z)?*oby{BjGvrWH|HFmMwOaQI|2J z5g!LEz-QYA%;>xdQHz$ISurNH5Z$`~*E0IxvGYUG(jO&VEh)dFykf(zD)dC3*H_o} zgXeE1)v68B^&jy@SeaSvP966$>KB5T_@ICZq^Cty#qRa z9A10xWz(A?vFBIZ&3^OwO*%FEkkfZUwzz)y?=o$9Ggpp*C<|X9S8;DPtm048;Ve-1 zR^uRUmsBn?yK}RytU4ma-LOx$u~+=-DYu)!;=yVGNO3;isLyX>m{jI}5S)l|2IVc? z*3p(Iq&n|y+;pbSBTyki{xTpM^_U4L&e!k!rm>1VtIr2Etg7tYmk}}5na~K-i_?66 z&)CfURyA)QI(@_LR{z8+tqOGgTUY{%fj!T zcMEz#w$BZkx?Mz4Z&fdl0;bJ)HfsRbj*V9RnpthoLj?D-aebFB03zl*LF!>1q@>Z* zg>XeSyWSbOtwJ7T_?9jJ2>IyTuD`7Q)jOPxI`uWkcBs8Iw&ewUZxsNOJiOpR6Md|S z+o(EN0#jc_3qcukDLOM^ty_U|Wp*YG_etP!&CgE2BO6H`^1g^Rgn4u+?Xn922(+1A zk8)c_*P1N=N#sL-Kv3i1Ol01~Y#hEdW0%I$FUhaEy6ObE>uR%Q;Wo2N5Z#hRD)8eBA!K`Xg}ma#@a5V3hMb3s1&Tv=Vbx_AO+35hfX%GxMO+L_80|aUO>Fe z2ofE05h12Fo&r5H;3tS%XDct00d0=S?TXFj$8%K~N7ik^Ps*)oe3YfAC{V{waAc=d zdZJeU3E_^7WO`u1v!J!X+ZqqJqv`!AJ|y`k4=c!1Lb-75GYTuhn^6Qygl>hU<#=?-Q02y zSCWdG{NvS z@msxpD)X{;We~jGxF0dMLp*X_H0(fd|dvM zjd3{p5Dz9I8CT>BT_e#QN`_L_Cg#Tsie&^Oc0Oy1HdI{QacJM#OMo3XG6KMjzN>a{ zd&kaQT0Y4m0g^iwYyS#({*?I88k>Lj7 zneYeb800v{ba??{`rTk^70!JJ{@Al|0c@cKz=}Flt}VdfpPYa!{TIsH)-w*7Mg8YB z7C{iIob(tmmTT^~Gl%;azM(Smb462++urq3Td2@32{aQUOh?o&R?}UjxF?{5YoKAU zqhVt-dJR4MN1#Iw^VylvE04V=N~crHS(nf_scSJt2aoFLFb0)pS;Y z018#9hqA1C&f{9PIX{t9YAecOvqMigqb z9Dn8~-z(If6rsaGoF=^~#{lB70} z+#t!RFy5Ymokpj=#kDGSXg}G|5V;9UGymxf9>#3u!sRo-xzAnZ0p9zcw1 zDRptV7Up4mntKD5YQ>tTJbTS$xJ9uNw4HkUuqVbCT7FsiAyLU#6=$VW3I~^L zzS6pH1>|f1rr$WhI{a$%w>}dp^^|`C6Z(ng#R@O_6@2Mp&2PH_V_aowTmE{o{^_fP zA?r4PW&Mij^RWr(xs4)Ki#~O%HK_g@gz5tSE=6?^~;#12qN_uF}&dL z>R-Kc_pQpkY5TB0sz*I8K$QkiJW2o(&p!6J&DxhZ3qn|=ggg4W_Z2FRQWf;#a?}G^ z-nl)ap{}AO4`PDcAmy=9f6KqJG`@2nI6rmKuwGLhb5~vB;voj_^Z438v-87~!g)!h zK=F@v`6xX5sBKYI3f{Tl*w!l8S{z=c%<358t(OXTqKSGPL3~z1MT9`*A>Rx>7+rAi zHyfTno(*r1Wf^X1D!g6_0=vUs<&p5ZAmNj{{^nSh_EtiT7DDEa#UcsrjR?W5z=w z8!)E&G7w`%hiu6mVs3?1Jo`tc1F!RpfwSBByP&lB(d4;CN&M6S^ecO!WJu;`Sg;0m zHz70Fhn=Q-xDmacGH0m(wwj3{4`!^c=2qQX1K;^*cZ?sHIJMjVS?12~2C8P0LjU(Z zSR7j32YB;x(qg71U;HS&_;mp!(i~8PIS@x5$mvkoqBaP*d}o0zUUE?A;6uQ&GKS*syHT8{(b zCeH^qBpX?p(95^LhVs^49%1}rx$tB*7r0n2$UcaN6tZ3Z8oA=@z2Sdv0o$w-CgU8o zU4i^i2|qj9d$Vq?Z7j3H=vnltL`E~6d-G|HjR7F*-G2O#k!)r~5EEe1!ppzNE>)I) z!`2DnIkp1bXd0-ad{@?zmd(Y<$K?mus&xE%^rO!?tHqYh#=~|d86XRj)>LtcKDK_8 z#lKbYUQh-x%FnC`VkV<$5#<()N6J{R| zN@vI8e4_z$Wt9Lv#~8>Rc8ORN4^g@};f%j&b=q)io~%zZyWo!*5PHsoK6gnTy&o?+=gQmM`A)fI8G;Ty5%we-r7Fu7LNMnbdI(VjF z2lJ`9dx1>58&soa4xwh?@DSOsU9ZZ@&^i~)3Cx{H%lY#Q)fn8WyY9(9twCu!ls?;1 zLzmNMw?!pJq`k+;aLN+tt zsZX@ORW)R2sf&n|od>e_=Wf~Uf0Lg9XfQ0lpwn}xw_kG*EJB^w; zy88IL1LKvRrABU!Nc}7&hT=F-F;93~@^JoA%eh}i@<>X^KEa{U?`#&@@1`Aw8V&dF zO6o($-Ot~^5exT#C9pQ~;agh*4etaT=mmmu{x@+EFR_Fo@4o%Zd$czBu`G2@Ya82<}FQFOP-)sliV50^&@2Bj)SQvSzQ-46ma zB|YGj9PRyWOpes93NVPXFJu@W7#~xb!kNjOAt30$ccxtev zguP4-Jbt9KjAml2?6-8*lh4Te!Km#>85 z#&;;kCdMnou;*_^k&J0Hpg$^n_U+au4PW!fHVM zkP<#ADJhZx$r%gwZg%bf78fY_nMoV9JtbSn6sX)_ddJTl0#RXMz}}#GXvo&Wve4BB zmnd;l^GD>62~ZXxzAC9AQPp;3Lsq_Ok(OfM(v>j@dv`bI2%hbq9XwFi?b=X%5xjK( zmO$ZN&cRx)*wgjo#m@CKWxyMd1kcx031+ujRuqdDiGzk49k{UYE+!@f_L+;ZhO>YezpWH5~(p&T#eic=oC~y?NJu)D9RJ0~mT=y`2n4Aga^g4UG znK%GG*dhUK>ql6_m^-{uOzqH}AlH#3V6#4p{qhBUIN-g_1fx!S3vBgL|&=@rH3Tl4Z&LJB(yEh z`h1uks;fPa>D4Njf#r=i^e8Z@DD7h%tM0f=cjkdxbwa@+UVgK#nL}AptR|k5$!~jc z;jnp0&&T$Cl9Iee{OASP*OaA?d7;@O-{7Z1a}}7L05-Fn2J(4#MuXuRc@AeC9IZrT zz^W^yxw#_4XxWPJ5UV*AmZ}YbLaup})-}*9%<~5j`MBAC`)YGS_~n5c*0#wcrS~Fv zey?7}?SO4t?i>48xZZZ;+LHFSE$GrfCPr}TWtVecu-j#=HN?MaYfeRmm|pNKJ!wAj z7GyAGR0F7gweV4~(W)UcEgF1iFHtq{DJWW~&IWe_e?r?lvjptf^U$~iGr0n03CKd__g3so<(%%)2j_oWr$Kc+GiTxDH>0#md*Nc6#){4b z7PWcjC$M;;VGM_X1S67MODYy`dI2gQ%$k?5;QH+s@B>6~TLtkA6Estg{uJ1*D;vs7 z2?~MaA+NRW&^l}qx7$d8SG`$dIeCJ*rV16KU~$LrziF%2?RifshQ|@Cbg~_!2pP|w zYEK$yZR83ClMqBJqS=|t52|jf5PhQl^3sgmv(FgVC2@}zfG0%8kFir+BNpT<6U~}8 zX_$OmuE6D2CNp0ZnlTklpz!qHLO+nhZ?E5jr`uEPfhXXaMr;?C7VJYe_RZdh2+Onf zD&{(n&x4Ay!c`5-y~#`PF=;!nd=5F6nuk6Y%G0lf(9CZs?7QVs?w$%!8IcVdXTM9 zf5P*sf2MF=^o5M9R0lWls*k|DVW!$fA$c5=a{mSeCEsAo6l%T z4ziVAUK2M2Mwq1yD5)o31euJ-9j863R_&e336b`ba!TlH{05p8ks(9XAdj{=4swPv z8gPRA)c?hpfO4(TBfwV6;rJBkb)Q^en8fm0FVA^g`cx#m`0syhLO%%9{j%mTX5~U}WfQzZ zK-?eF)hq_S^otxk^8i=RpiE>zCQzp+t(A%ulk}xZsJpCve8X+et9D_j>1|cb#!{q^ z?@KF5L>@`4lZ+7bLeuRn7Ylf0Wjqz#Z6CIZRNaHE?_tKm04S4vx-K$zS66?)%?8Qw zmuw)*V9IxQqiZps)koT4$a9Re+|8b*g8dM^S@DTaZ{Rb1g*KsUmC9ZceoqDY%lGTn zk~P%{%O07O`LFzK)29!AzWK5=_TF&)7jmM6o}@&cnMdvJM^BKbM`oQ(?ZvInmYBbF z-G|QK*TtnzQy1{XcOoJTwnr%j(B@1;x_#A>D)D1O&R5^A9g+g3XaO~$h&whgYnoZd zSiF`-yQXk$*)(Wfsg0z(l{nZaqS7PfGw0d8X>jnN^f8 z@7^A>IHk+GKfcs7cc3m6?@z0H9Z+Ke_0?w2*lKs?r5n93Gl|(igjcgmnPS!3J_SH$ z6`oU?)B}n1(Qtb=B?bNl|0`WxfDoaU&*9SH*c~G;MCy}PQ+oN1P?E0t#!otYZGk~?oBt@(*obZ z8UyT>Rd-R)duDXaj+woJD188QMi`kIw=l_6T1GiXBX&E``tA#h!}fpxY2z7ZZTj;k z)MLm#9?FRvTOaP;u$0V2C@Bx69mG{9xxsi2{(-KX{f_#_RtIpEGfvjn#Z*&V>&V!@ zdQB}vfJtbeG}f3~Hxwc(zPElo*Ie4fE*AbF^T(K&F6^OA7ti1op-t1-yf{c&aiQ8G zeh?oRxh&Y1o&*fVFG_iXH7<&+)xCi!1KR_k;Yr&&W@xYgH7FqSPECK0)s76C2W8+8 zEAAbOodzp;lox25&PrVX$2eBm*(w%5F~Wkas+u)Uu9>^k{*BFH!b^L9|KIWZA{8*E zDJFq0lY69bpdS3+7?Jg#0F^2}Rgw=9nP>;rD*OeINEw|5<>*Vxq5{d|PJc%G=Iw{7 z#$1rV76L@h?f|4{D)xq^lq0yD1%zm+_X~t1f{jm^3+>px091~h|s25x22tTw@8(5fR3L&K_xRO#>DYCxG|L9@zf_$pf7U^fs@{VQ zcP1NeG-er`jw8s8E_hgG{!ny!Ei{g8;vZD|b<^eF`lcNBGmVz7RzkcYa6$3lf1MIx9wY`2XE~dl@YSo6)E&J!SDTkIFL%~la^7?(zj9mTCD4e!BbAwzfnrZM zWCF>rrEbeO?$r_Z_tZXjKwPR!Zs{-$+@Q1ncL_k|{*w|YCVdT?T14?* zZ;R*#=UW))p`wb$#tCO5b@p{plli;14lnm#L-o$L=@0+;9lzgHy`*Bm{%ghOdm+}r znm&PBhU#l}tOM=GJukQKY1-K zyZ=biHq*2XFTwl?;E)UZeXWBNyUR#G)@^@bc2!)bxQy54ij_W~|*A*X8lI)#n+ z7n28M-qd_axL6A9L$GAO#Y(kQ&3f`B`N$cgD=#`S!q^_S zTqKcNOF=`sft$Ma572eelZS`hwr1IYs;hf!spDN){pZ$Y6{jq%PJ7r2l9yS;^I^4> zZ(DHNCU-yW&_ezzas3Y@XYl};{|CnwL9IS{N1-#Nb0(0Ib#;dPC>M(o7iaGN`VSw3 zzUhD8CkV&t==-~>yO+$^HKp-f7|7ROgmS6iP8Zh584%o5!~O{0-y{?S*e-A z7UseNC|t=A2yA{i;W9jiIWJV>Z6s?kvOL?CJA^#&KMnJ@dQ<-Ou*zIWFgr8hNbSOM z_RQtM&JKG=HI*-Y5tw#{AcZf!hcZx-sO71ZmX3eI5pZO%qc5AyP7}W_3G*(3^c?d2 zORH3Gds=pRLum3zkD||hgfDZ{Li|?OzWmljY(ZScL;_IstL~On7V%j!l&@kZ=w` zg9B)iE9!48!cm7NYarDhBb1}BN zYXqY7-zQR1;31c@p;Oq<>7dgmc_d~-5;s=AuwlSB-Ats{8py@Bn&74=k!lqS-@ig= z5em=(hQW;GCeH6ec9&7zp;Y?JrOm<8wu2>C$B%;>$XXK9Bg**S#~K7EW7Ri5mAQG! zQ5XA>DYDT{t0_B#3a5Y1s){!`a6ppz1HdHK9aKRt4|*S>@pxv{_0qSX?%+JGs@92q zD_^B^Zu1~j0P}u{9A9?L`|RA`QMH#ENiQFr$IlPCya6M8g;F_%53a{1;&q@OC|B%X z3sQwF3{1b#PI#HE=v6I@c=C0lTW@0ytuzJvhbVX)JW;of@~z_AO1e|nNkDQALF^vtv0lQEB~tf7($+5acT=Q=Cz+2pfFDJTJO|q(2QmCXz`h; zLV3mwM)Y8UENmXKfu@T)J$ck~^%}1?-n6&+1%a&4iJc+eB6HC_lB2gIXdq$%kkZhz zmJF_~vGL`Q2*Oe%3lmZ6`-(e10=AGpo=P70>G0ljLQ}$-al6bUL!N4-u>Do^N;@dsjOYRx>M21QLK3W zKc&bbPy};F82rg}6R;3p7xnMlb$7ipYX5h!JHOD(?{1b*XE@YOD;gB&PxUCeZE<+h zW?F~?fXtz6>ZXm$s9wil@)gvVMT2|tZ@_7fzghSeH3=xm$hP~WnEXFg5dx<=uBpo5aQn)Kw+WYS$%PXRR@;FGP0ZZ9Ix z*K!=(=H@;z|1lHMz=r_=)+lg7)N@3;7w24>Rp#0E(O_##M%YZ`NO6;g6tS07iNi4H zH{J?uJiTTFTtvx-dD)fsw!^ljE+6x5T$<>Xx^ATT8syQco<^DCxk(GnbUcAlQEHej zRl)SA_4%Z?4mUU}x%S9m31yo7m-#^x<16BU)=Hy*h|In}i2wL2wu5WA-w#!L! zHeyK8e#@E;#@$7fT0Is?y0yMbN7eW;khIiieysJYse4O%E(j z2F?dp2jW0C5-A=!Ul1vokEy1m0c6yN7yL_G%t`}>ch7$-(}dF286IEqbUKU+T!)a? zZiORzjfGB4#pR8TJzQTV0Vw#B?ujht?w|^3C7!4?bE|JAxE@dAMmH`>;X6V4Om$iq zX!g_bBY5*O9|ovijOvJ%XuZ$qn%tV@$|CzzxUzNjt7Wzgs`{CWKeae!r&j*+T5v|Z zA6o#YPE_JLfxMYlFzdTHELn(gYKKB37U?xzUJ-ty?^N#hk-B9}dT5C*kqb_O8n!w> zU3#O^n)ME>#!=KGre}fG@S@jHa9Dg(O_GFgC7+;9eH8hTHAmTOGVxniySuDVHS-^@ zdo5Mti%=p{yBn6u2`hUpQx+chG<7BpPnSCmDh>}nf?A}KlEOg=q@dQn_~Q*#?&`ml z6xPGyczEbZ8I+q7fJLQa#q|WKH%)o__{es_=01s_#`AGbUz2fm7jtc0lucS!9Ko%o^qe^@+4&2l;>*4;uW1GsIy7% zU5E^W9hH@pxv0XzLh}x7c8L7+oH#Q5mO9@OS*+q z0)`d>Z4kfJR{GGhAvImTdBX`_B4|H#-{$`t)17knErcO3Pf3WHbJxQh&FpODak185 z33-v=sEZ;I$3}W0K)9 zQ9Ajfxc2FSK|gg^)}p#}E^Z71W-?StW0bhS=+jJ}w-={vPN#Jb7D2;d`8DH|Zi)WU zNKl_FMWfO4JEwbV|NPOpVQ?n!)%qR$BNiNGOS8l&Gn$}yBl5jBKIXo8pw|=e{3LjD z1UP>gSm2D|@P#ud>Nu~(UafyA@HauanQ^y?dHffg`Y>zkXA>G~an_6%uAY4M(kdB- z2)*Ei^9M)qx)wxY=OmJ2fAMY*=R{=Ca=<0mw||`m6T#DMWxWPJ4O)xI(CFkcL6wDp zzovY>7JPBZNpZKs&oyf&X|}O>42{-~++I|EM}o#K=N{?nXVCgNLf~|l-il5zt*=Ce zpeE?i`Yp!JaZrxHrKNOTo&aXHRtE9On)`enF*?vyThW`GoP6mVUw}V7``Z$jb6l^| z0jE~NThr*hWfbRJ=Wy7bTJE^A=^&dYdi=_#PJJ9NVu0(?c^l)D`RD^2^w+vq)BFk& z5F0ND(7*jU37k*&1UCb~0uN&OU)2W=%QsS$B=fhi?2VG3;kAa(*?sE@*3|-&y zFTyGosr}^*Q@#NCVJYpB7jJ=7Q|D-9@*%mhQ2TSmSXj>xJkbt`D$SfHJ;tAeDlnf} zp?Nj8&rYT5=kstscY5^bNnc;zx8~-L4?fS%J}P)La$}*VjMl>$XpSf>p%9C3HuPbe zbLemR$;l`9!5onoIlDKy9||oxG8W(T=^66}>J7crvo*+#m3MgGj+FRlXx#{=xHR;u zwZ+(-8UA#{VLXIn)n8gN%oQ|B%K;aCX94s`IWbu{kp!nboDdiiX-fX4(5Gf_NAA|# z?5x+}$?~;W*@xt~ zL%`%OX(cc+3WxBTLsWWi*>=vs!NID!x5LOSUH6GAK(_-rV()@ddsIkYr4RFbt~uB3 z)@5?mJcdGDB$3FR1WE^E3E43md=Wi;`~3OyIA}JspXk8*aCq-UjOD#S zoADiUS8JbIdv9B@8N*Q}&6wJEfr%_QIhE>t6gZtwTM=41zwaJ|x`ixu;W#}%Xu603 zwcUoM>+(HR&7Ae0?5Q)>c4xYwk$Sm#Wz_nzoymT3Jn=Penwy%MIN3uc?*;l)+MOBt z^5skSR9;@*6qqsj1VZQBvG?+nr6qIL@@$ud2HF0d)7@NLT(}Z)AA4KtBzP+_qr-2k*?lOM0Q_ ziKf|(B1eMhG#V|}BJGmhEOnRsdfNC4IO5I;lXsbUfLMr?ob_SM^bU4BF_FlL;Rw=- zjjzpnG5M_4Ifr`)$}vb~3#}V3W-8~|`;vB`&_|$i?V2GaUT-)GDn^WUj#rxw78T>a zOjDI=z)6_-R@wYXOPh@J;cP2?&xDza2Z8EhtUJ(|ZHj}_k{J8IbP)RSsv*6XQEY#N zT?am%afKi8Ufc%EDa&OUwD0NV2liD&V*O%=&15WPcIfVqfK}DZ`vyn`6**0?Vc-mR zKcBLPltqk}KnAibPSb)SZPxh21^JMYi=^ZpB|H@aRZEz@zRrIwEvUD=^D^2V)veubqNMt8Is-P5ZH87mGqf_moXlu;&gAo|YiK zbrUPi+t>x=eQ7JGWk7}gyohGuB_)qn@kzyIi!4&dLey!=CA~$ zm(g8oe8RhpTh4(F6;vsx(wWv1$%fYTA!F50e=02Q2k3_DS;-~9L?WJNH5E-M6}Xwl zfcJ1B(!?|66jfA=^l^Fo9Y3Z!JN2wsam!T9MG%^h!;^l@AWoEL4;)OBPAEP#g?1pA zrw-O|+YRa3G|s$Ck*`mksR{QSx>EG8(VqI3f+ogss3xde{6ebUlz!>tOv%O6f>RD? zMkNP)Ctz;3W(~lc*1kwLvZB-Yhmy!PS2Etsky(;iPS5nuAIdmKZEYe>&B0*wk7EY| zkKN?nNl;oASfO+w$(C8e1pmPs9HHoB>|PZbwzeA#^nlI}Z$2(u*vs-@aCg=Pr$xEG z%NNg1y;ScRQOk^+0ORbyt+)Df^Yao9xe=C) z>2Js-N(k_7WWm7T$+_N47=pb1wkc47MnEswi36>{?agLO)yYpssSk9n_C*LJwyLP0 z6l2Q36@jxD+9`v84REJ=_}Ws~4vtmU2rB(T(0RrJJByU;FWP5qO`F)SI}8OM{IR`P zF%Ndplid)yq9E%4i?+5lu0378cEbL**)MufPCjf&NNhyEn zl^+Q7GsW3=4~`a(#DOW%JPoL+fq&fM9F)^lW8fWNAF)#ox%v6RDcx)hU(8Fhz5kje z)=yd-oh;#F}r;kOU^Lu%=Phc z>oCZzG1_Zqy+BvrGCUXR-I3;*i{R>9&T?%&q<{>+w3!~lSj@J4XOo~O_7rLl1n+5v ziAzualXBTgey(f3>0t{qGlH@PTtPtrxxgpfiFPpaOYN4(*Wu<&GU<1VmCk6_m-rNv zI@_n7Fkw&j%%tgHbR^zn$WEFocPx?!mAMuj0pYVSdVD8{w-Mqd8P3RVX3_)Mobec2 z6&9bVQsQRUCykelH2GpW2y~R1$Zp|);&F`RvnR4%U_|qJS?ka>u7dpjTRv-N6knh9 zW>AC3mEbgaTNAsZUJeZuW7Ey+dk7}G$E|lMg8rF&E75zO&CKR)z}Bbf4S=ckyE-zF z@$pHSnUYo9o_r_n#tPn4_#9|rI&BN~`ox>1lSY|n6fT66-I;Int6fYEzrM2R2j%nf z8A@g)Qj=v2PpdinOo6?nf%xK&rnh$(blR@o^C(MC1EYKL^SPfsojy&tp}Tk{LV*EB z-196^D1`I#U@uLk7fMPgi~j4=V<7fMaQzrD-w`56+zG@K-0^A>3zJKDZ z*da}yGCNDL0IPS8T`2Plu%kSl*FG4GN5hWR%GYPl5vAg#dB!6q)6KEuG6ro58i!nU z4(HkvC|Sx2B{6LQeq+_An~mlSg7m5fhi6Fr^>uY%1lxxa3ZJa?*Z$Gzq1y9ehmfOP zguf21Pp|<#{jEwhodwEg1wv~%SO&7oH!8uR@G2JC-=#)EzSmc2V%+y?#+U4;kNP4k zI>dW}&#(FS);vga6Q|B7EBur&jMIVKB-sdMiUINNk{K_A^GWZV`UGtxLN3c1fKwG6 zx6(SZ-zxJ2B&MO*@_=GdUbofc@PuU4w&?Uk)W z0;P;mmyDMS2$d2crPIoLf7>RU`UYyV9f$+X0!8q}FIwi+S$2QxJ>WyMgF$?|00Sl~ z*%<7S8MqJ@f6dLK%{iu9>dNo0Fn9SnCmP$>?eEnS9! zWGnP-N^*bn!WmRjZksP+hR9dAAhK2C>ea89y$t7^X7Wk+tDnNAas7@e2K|aY)kd0O z7I8-GOBIix;R_L+IZ&YdlNu?HSow4XjX8N<#P zvLQn?6|&CjnEG2mF-;yNsMSX4_E=SeyR~_zfR-pqDl(m&8><$aTCHDryX`v;yI)-7v?r7VKn$9Huq+$6l-^*axjnW1wAVFe8`4 zO3OKpNUfV_=JOb0y1^lRM~-Q99awXnGZJ~R%5Hh{#7qP9DWn|j$J>ynY67U=tHvap zCtOK=nE4syOVa+ID9MwzmyOPt#8$++{Xg9gEc$$(tDGon8+@AY+WpG=#e zV9xw^s#;o@H#(saJKI;U^!N;J<$sp^ExD+uNH-?j$I`E*%#AQ)i?6v_pe!AkEg_td z`gaeWzVHNf6`XQ9q68&%u8XQJGq!Hof19!Wv1Qr6Y*?m+Ujgu+^yPq4ZA-J92#vA0 zR3vTmV=7U`wHjy`7gNbK9LE50-Jb1%7N;=+cwv(kV1#h42UYr^*bzN$bqvk@H#+C{aP-yp#PnCmXxRPHVa-G6SI-s0s(4Qs@7&L0dR5Qr&dyGk@O%1! z_b%t%RqyTACl{B|GaZ+HP;b=V$S>5#B!6s_pUjV$R4QX1CwtLo_cG{o`W{AUwBi=o zc*K&xSd2qQptswcMqrE=uEIx6+3b^UNuMZT-0?1M1{(Y60tOHIYRoXg`sefVtIu>8 zNLS={@N*qik(1+HLu5iRusYlgZ&Dt)D8)fZ`p%E;I0a3c!_SSeCWFbQZo`rlo1aTzK+h8Qtw&9q{I!5X#8YS zSM9v1$%{AkMX>5FCSva@wp$h&Dhd9)H{YUYw){A5KnR1 z5{sA|i2}loq~0N}T}Uuoa0q0{&#)!3Az~FC7`HwyXr`wx$L|M(y7Dvp1hn!KQF(_851@^Fz{J6-%#v1m0tKfQGZ{!xTMm5K7Y zZ?)5Txx2rX(`7DgF*`X0Rj?Qwq!!>YhWf_eQ;gA*o{S=+_%m(VY$k@4?Fclt!F(G9 z#uTsDUS45u>h5<&s$vK!%_)JEOEcboQ1wK?S%crN zJ}$q!<2#HO8eTaV=SK7wyU`%T{<|lg;H6>&G>rJ>*mnDlJnX|n{;`hCNq6rX491_$ zx{bG#sIx6vYMxzxtIap`+SL=wO{L6GP7a26MpD#tb6JPHGo#^*;N<5D+$rZCw|m(( z4{f4F(?34Wi_~%WnqgQpC>_6e73}p&ZC67f1H+!u2Tb#i`tO3|FC918fnIz`(B(mnGx-gGf=Sayey%fG8{vQ_oDmB_teB(6$_JZ zNz#H7k#p~~GLnrRQO9#IPHDk&M$s}>0aiVEyY$1!QD??}FfgAnl$>~)r8`g!5A@xV zgj2)>JS+M7zIFAYOXI>&UZzZjzid0IhOd`8zPmJ4lZKvlh^PC8^j$9aerc@49M!BT zHyNGXTqcm2A?Rw4aG#IX3fDDT$Br9S0g*8q@Fj+?ex>QrsdP$Q&_q!%^(NIWz9nK- zh4?z&92eE*yvoNMgOi9eEHXr zF0=T9Fr@pm?VXtf^Ran|LCg>m+z$3L=wwqsLMt1Sk!dmTkG~YWnz!teC8eVJ zpM}r054Ft^OSTR9gQhu;T(0cDR@7?9fW4))f$I5c5$y&U$}&cufGu@@lGj|#_B*A2S$F6 zM6*agh*^#MjT*rOSnhrr(RpZ*5HA=U9=J`=J2s_%m_I83%>ob6 zS9Qhtz-m}w2Anx&7y-{qJV(r~n-htOr%&(Gg3}(Q-O732V$=2UUM20y^|7V7F-1&c zuPvO6#$4ptQMS~UNu0&>hUP3R%=g%Y0RP^dDOD?hH1f#SIiXU_DGHLFWbe3DJKuN|@R7}F2+sln$UvrR4% zVu?kQr9+8I)A6N`Vq|TMis^HillypWx<4gzH4Y9z z^M>s}=)TDay?`_`+_szFk184%3ijBpeLSdr6`jer9xu%vOp@!8A0tJMi9K&L zj1Dn-F(J56zHKl!)59w1c+oR>6A6oU7%Cjg2yBR!bqJOpaKlF8=LU4|+KMd$0vC5dmdIcU z@u>QsK3SR1jqK<(njQ+X8osQEc`&T2uJBe~MzMpU+i?#z*k3yGhtbf5j`a1{=7d3r z@|}uJgSZo9|0ff+mQKGlGp)^_skPtKI%oFj)d%am{J9f3ZM9Cd4bjpP_s_S{?J`rT0k56b*5SZYKbc0?x7E5!M3s5$X;F(~paRKQ9(_4BPK zc78d7M_Q>7E8JQ%WUJc{$lgMVP2(%YU4$6)`N;8tLa*j?BtOwEYd zmPcYO05A7+UOMp&K`WY6M~M45)2n6{z8`G!+Gk+@w!z(;1n#!SJnV~fi38C-*U)cF z4nXB96^pqAde<;%n4X^1#*3N@#*{{nY=)|{+OuJ*1251UH{@W?m!(zTYAou>*^ZN?JhfR&n72XKcfVu!7(NCZqsTPnhN zP)I>n_LgOg;`jRutmDEq_`+GU+JXCJ?v3?;(z-xlIMTrWvYA1S78)a;Sw>$f040eb z3nb+>)@`yxSeXEN1oi_IN1Ao?e=M>=^<_7L@=vH2xMn1zjiG0?Xk_Es5`C!{!E;46 z!-s6&gJBdZEh=`Ct7lZ@s#zE#+7?6p z4S%WKw>p**y<_ygTd-wdxJ#Tv@a_0`&xU3>vpbaCuey2x!gzD-aiR3|3)r~z^bCGW z7-kp;Sx1POTM?}lJ(gx&0JCp7v9s}DK+bEqz!y0-J)x`_w_dxABu3VdFmOS1ku^^) zNUvpwzrNxE(c>AZQ880mZT_H!l(j=6)_nxY+SdAf!dZ}Kl1-Fj96AS76)6mff^D<_F$^e`8&t?RdFty;931ZjoP zKYuJKe)5=~bLwEmlI0Hr7g%SI@yJKh)B>EN`q2F%&iLc;^mYFx=CLY8Hd6SccLwv1 zW_jhNTDz)<9^2GHzUlry!oEBn>izrwCZ%0cmMqgPQb`n9vP`#>8%o`>ui3>|!&rv2 zk@ZH%HY%ZPlWpuv*%D?#h_Q}kvKz|`#_)TM4C8+4`}?QI^yl&k%#(ePT-94#p;JwXKwa9fh#WO3|vzo3?byDVi41%+`=ZK#1VIL_akCu^P^0! zx{0QW4R-K{&&e}jV8q|M+<<;7kyr#%O&1~``55y=V$uVwaCQ=jD)GD2-zRlyq3M8Sw?%Q(g$9awS|F+>b<`Kxc?eLd+ldUsO*;fKe=82q)D#WV1YxT$w=g>)eaGLkPR1}v6Jb|^eO7$`{Depw zk_YP068aJc?N_`yLih8H#nPzjHDUV>mp{AZxJL8FkU6#-45h#FAD|~T1anPvjnw|( z6J5SlZ7k47o@Jj>^tKAvlTQr_fPrb?6Vormw?O4>;ayo;g>aYep7UY_&<(8KcrIyi z<{LF)IGwr@{m`4xprxdM^7W6(| z53#;ArXTH3k{bS|UYPVabPytpgyn+kTWJW`mloUbABOeQ=X}I(^u8Fs6zdqYG`gcc zG%_9I@+beF{ZH4e75M1Qi*kZc4N>|`JSwW;NiF>h3a|xy;Xfyx1Ff5dIN- zr$%i7M?tY&^$4tMlm*$z?M-HlQ4Uyt2Qzrs+p2)N9Hd+<*|+vzjZSal_qcb&Wm5M0 zvY-1|2a;ts**_dU+#@f##@i7k=h|hy{(4v6i%vas{qsh2sE3FuNW@xjg+SWcTP4pr z=~uQg(fl9Z767oa3-Qu`j*I|S;<|j}4+mb3(!xY{yFYZ3IA02gYH{MYtM_u2R)Gyh zt2DIWSGIaY%!=8O6TcQ!rL%_Ik&+1wl&0*cN8`^U$M2)&x|YU}^BpBPV`@Z~*>T3S zIf-^_#mVWv-1n~ePPx|Z*sl{6xy&io>&X7};=8poNd@4!p;SH?^#5lni*1t)^R6Jugq$R~_w(uIce6 z58F5}{A)4VYsA!^vo=}Ne^~ub?^^(O$CK)eX%A^p#LCIbxU}{7v?K7K^sv64iUL|y z7uy)2CDXR3Ft@Rvo6q$cEnI|pKZu~ud~#X)uobZqDn?}g`F7xQ#pt`dNF&{}di9^* zDDmrWlkuu$N$?ldX@TJb!8WC69)9IKET-c%N2xO12rL8%h8|N4J z9g(js)~R}03g_ei%RC?o>zXN+5jXDE7Jq|6Nd(T}qJBM_5vKBraHewMaKkr1X*J`~ zUO}aleS4}BcwDN;{N2w*Xle<4`;Hd?!@0J+MMq)7d4f9-`O6>Bj~xM*qO1mQXDkXq ze4{08XHGrIz=`R?~gf5s1-+gOAx4y8MZ`Lkx-UJ;adT<-oZZ<2PD?j1GVn zUgTp)v><&ufa}_o75*t{)5Yhj{6;$L%isk_r|P*pktEIdz`Pv9kFLlnK@~zygTOE6 zJ!pr2b?j0sVq~Gs-9QQ#O>S5oELBjd{&FisvkioNkn0-v|DOv`?hSLb34~-I6i5?1 zBUC^0&;q2ynS^M>2(hV^FK;RK9=Xp=d|W{6NLYS-Xj0x-OB8o4oxf&D(!eWZl{DcF)me3lmBPLWw`Dwl`D^ zV|i^KWcX}W2|u*k+0~$k6~d}4%XsALR_MO_zRRDbmR_B+VRb;fo_+4t>FyZ z6uChGM$NK2<^|`(UvI}VFfpT0TEsSmyc}FlXO(*s;&pgNCHGdoHn9QpbRQ&K3sZGI ztikLfH^83kF5lh3jIOi+A_n>8v!zxJ>bYBqdr`VBvjT#mx;oKh?Sq|xgFtIL=Y=f= zw0x-RE+_sW+f?Y<&&?(DE4k`;XG!g#V0XwL|9~I+$1GXagjAvJJWKF=IH|&=z{X?NCj1 zL}`L6C%Wg@>k_;*)OX!<%#uM+3&%@EB21$Dvp^y_U%K@&@kh0f*349J1&kp(zG(Ch zZ5;<8VOQ4Ip;10jVKQpHX_Rr@_5yurTTX0J-;Iw$87fRWH0(V(U8=#{Io#Qr%q^Tv zQ;!ROLa*{>0K7;8!1;eq(_6s-V{w7-tLSgMsCE5Izy~AGZM1!DGsqM>qb+G+`h1GG z>#0*zg91&XZhK$FssXhlKT?Vh(4FYi(ut#k8nBI*y+AM#nrAU}TQd}&OpXtfDXrZx zzm8h{LSpHs#t`j^7BwEIvLlPrNxXCVk5tlHJ9nO^yr7tujm5$Q4fLTiOQrb~pK;0Ii}{+rO`veLsdK|14jX+m=LQZvzUb*14PH7S&t6rT40qc0F0>N zKkoGqq!!3t$iIVh<)m8KXgtXWjvwyIv|IVwr}U!IL8Q;)&N&&6R$Lvv!#L5-Y7_|t zt#|m)#1=Kef6{O|r#Gr+J|jm?%LhWAaqFy(_QfG9l{5Kr1#Ug5YT1T$fH*zHKWiug z!nO?qVyPPWbd;S`?N!$~OoxziP%xMW1%s#k)E9aumqpa|Gz9=_UiOC93;dcSBP{IB})__NbH8vK zu=oOm#hr5wT6^f^$CJ6#xko(Hq;1&*ns#s{!T@%y53NPQ!olHq_bu+m;}e~Zx|Y~) zYy|_RJ}Cdb+w=nI3THzA4}d`diNfzjfQ zzoRW=@uL<8eIaSg3hs3saGMz;-cz+bwwGMz-C)mpY)(^sWl4VFrHS6oUZc5$>QR)A zBWTv2D`1e&H?8O-r`DZu!0NBegkZ3eP9Fu6ae9C0mKh+5h)Z)tlkz#omdixt+hZFT zyEfSs0Y)${963!7`(g%6A@b+woAsGt{ptJGpUoRU~o~fMs)U)Uu48s|{ z5kXReo29(B*^LI55TDpV`3&bLnq>m$Pe_v{cG8kLCInXm{Sg2w#Y|LCZWxjK%%}5D zPiYZtM$w{B3oz3nC7wi{JACu_eTG=VkWKky9^lH{?Lhrgo4GThl<)d!O3qx}LJ{%w z(eBy$iTs&v7!AKRN92dJ2yg`Q*oP>oJgYH>X8u8g6vS+B%|a^ z=TPf3;O7S);vt$`S7LluFDva!d+Tm*+;lm2D!DdMk`fLuv?-4Va@arLyK9CxEGuQc z`SFi}$A9R#KZ5SF;PG!~QS5b(N^>G<(HGK@$_QXa@2SnIw*$BJUFu?)Dy?Z~XEzRvR^b9&fo7cw4$Bn7N`~ogk*b zv_eAh2j<3XKk9uZ!^!kqzMh3qPHo@fON?R{bd{iYBIb>YSlhyXZT!7CyHrtX*=d*hx7dFH^13` z*`}~u;GB84ZsPe&Ygj?UV|!9sUgUFFW>a2{GZuih<*z~er4|#SIjKs04YW<2{IV+R ziITt>DPuSqQH!8XzpMI*mU{qNy71c0tiS6BJV-Ka)W#}-OdnYXXy-W{9F#c$sFL)z z^sxp-*h!K|h+TJEXhEWi{6|^dx_~j=MOSQFh5?rfXG3u5Nb(f#FE{9j7?Bac9RzB0 zBiie^2?+5nT&qxx6vZq}p|lVaJEz%dQ_I1q%O=zASg@)jTQV}~QVU~Hwkm-%3rFF2 z8&$VQk^&>qBXTB-Iq(%-&B`pnY!%jDXYBi58ZosTuOz++7^(_t6R|eb7I)Q~E}!cL z=e#)hQb$gzc>ZGt0Uv>arD2EYwK5ubIM9L{rk9`I1kNSuN84bY>)C1B%17!M zH{{rntY}tj&An2OK{&KM!u8jRJlbqM|Fb4mAGPkxvlDLf1>dvn-jVw|4wNSZ4W_l{ z&7v?fL-l@<>0`?S`jg4`Hue3l8?(>OBmYt^7T<4q2%nteMBD9ZJk5M&p?9$(CeyKp zH^w9@od=k5hWFC^jiUW+BKjNt#AN|8p8h=(4h&$9?^_|V#u$6#KB}c`tt%>-En=Z z14UlgxU`VhcPuP(PQXq?Mz%A#+u$Kc?nL^7(5xkt>3sNb?71cek!AhV zz=z|{E*j2dJJK8xQrw6C5#RF9rLUh^LOx4Hy4noAtV1~2kg#amrQVO19on|y`up?b zk(*?q>RYb{V}njbRJ5% zLJXwsXz?PSC0~qjHS6Ym99AGiMDk6#Q>WRsmBy(Fd(h9tlkv1^4tz?ai#k)qmJQ$%4=u*3h65__FOgO?e}R_=BKjR1l3O3c|L z=j;P4rDjX}G5RwBHK!&XF2>&_z28^a!A{|W$Vpwh$(Cf89w;Et)JdU02Fk&(D`AF4 zI^ksfT+bWDIu#9~tb zcK{>ka{E5>)9sQ=#Ro)BJgtGKXZdvk)>qYU+{6k;F4{qee-Jb4q`3-46FwjMymRSzM=pT)$j%3Os6YfaI> z!~k=p<1hFaJc7=Kzz&O@XHL?T>4{9&*$*W@oeA-JZGJ&|29xm2EZ)5AoQD+hwmj+E zMNhYeh$aT%>B0Z5M5lL1R#IbU3oUS^?X#6D5IKvJd@B-h+pIW8oxt6kTt(?AJQ^Vq z)dl}ommy)GpE6k9%P}U9n)q9zw77&L8@RRSW8CoOr3C}QqoQ3Pm~!K+Jr6M+Rhxwq z!$y~L0kX%x_0H|}Y+t1pG;XkC$2*|Y4pE-zLKdK!UKI6e{E#r{fb>`{$B705JyNkJ zr|Dn5-)+w!=5uqDCLpGhOBX&R%#_b9Ik~l??>OkD9Xt@}Fz4jj&EQP$@(StP25fQf zL0T=W>Q&NuZWdV%@y36I-|xT3drzgcm9w!`p+gH3hEQ1nFaN;=!rQ-WJDcZy^|yks z(XO4UlQA5cdeWlZ_vd7k?35c%@5Asq>YkR5CW3+v&_mSalHF~ZFM2Lwf&d-S z;oM{0fqN%+e+#6$_gPLYT|u>YxVso>9CBegE!1A5s1&!)RnZlnTLpxjo zydc-a5A z=~>qCWFERScZ?VWk-xo3*^qt$oL=qEr1x}l=og&I)NQM(PWK3w%MfXdPH(8R0UQ%3 z1gAp)1JYczY_ovHW3I^GWscuCex$mD!mZ{4?*BCH#!%Ueyjf*Fss6c6CxoYgE1}^i zUMa<}kJ6bZiF=lBHL!7oP5DXM0YQ8eqp{|3VBL|k$A?^M9$K)e1`Kb~v^)Xa$d- zFXiy|viI(!_67rhBn{o}q@y9Qohc@eW0YN#u2{Dtk*Vjp{6W60lm3rbthJj}(XFyE};Yya6tp-$j2EIaKrm@Mj_7_e9|RI%;buY z6^Kf;{mDR3_02($l&>ab$YRuqECji|hL zCP}2AgJZNMuhL1?wG%>}4tJ>XcXQ#t!SicF!r1%JaDL>Jg(tOzkfZX=sWnka7g@+H z+#gMhJqD_sG_0hAOaRsh6U~m6 zN#=dx*vsy=h)j{Kzwx+U)dTsxoy+js>KBXcyX*Ttb+b+OsGhXIxaDM;3}&QSWw~`F zeVlKKPCv#!yiw-|y-~C~2Jstr3|3#KiH#u5$9yj#^k&AltCpHiq7|)m%f@1eI=T}_ ziOYEzerD=h7`uuY$O!X}gR24`{lMl%oz;yCB6KHahtBd&{6Wc07P);%c_sHY7)Sc^SZN`3YM03)3}DdL|3sWdk3&=sBlU z)IZEeQQ}~Yyz=e9+XQ9wP`)`6x;~sH$*eSr8gfVKi{&j27c|xW8VErD;la;_A3R#Z z1MpI|dfJImMLP%G4GPkY;5nJG$qJg(#;*C-_cAE)-?^o{Th3HE^D{u*RbikC<@~d_ zu47;8ow44Yitl?g7j$qRR3_}r9KJ75auNrm-#RURnbP`OaVY%~v4}HBzGjNMnYnBN zqe#Cg7#mm5T4lOd73VKUJ9V5_UM}dq5Un-b9Fc-S+j_1nzoyao>KZ5+PTk2U{Bk`3 zInu2o!|O7gsLzTQVTL%Bgm5r=_vJwZvz0=z$h8iR)R2<_wxgOuX}M-^$W%Lhwz?E1 z9*8Z=_DRG?t1KfcmkpEY%=KloI;SlRH zx?E0_qS~%L-}My7dz*glygjmL;B{Rp`p_OJ zifNB|SuwUO%P!k{ShtFW%o|9@28~TTnpX`7NyNuFbeyl&T8GfvAfAVW{?Sma8r%w6 z125dtEX=k3Ze!32MZ2`}Dx%XM;-~w(^kae) zIqTbeM|Fr9e9w>HjU$|&i)wUYSB=<7lUtX12h3Ieg{5qJ0S&(spK0SB$&tdsi_-Cr0; z=PIkZp>brUfkp#qi9;rcgb7YK*up4U>&)92EPrNFAYa0Y|S>@jRr8MX2<@yVnO z|JiSQ8EiudfGH%?o=!)&G<{+~rK!gQt~Ayz`)#??ok?Q1=O`JFmWJvVv|N3 z6kbu~FudS&%=Phac={hcw*|jysX~T`UvOGbTJ!8;CpQzc>Pf8PW zLH!#2{p^9HIn$8^mBs$(I?n3OW$wY9j6|F61hix5eX+CtrNV03-li$b<{Iq2T$p|$ zf1r4j`sJRoF6i#)Ii+xCyerd;(hc&&xq?3hvWUOzSC>-Pa-o zu~oT0^sob8V%#8b58)~Gr))`w!#tRb(gx{4a>v5m;_mNGbX#Cx?6hU6vNMLp1jCY; zY=xB9n#EDrGYeDZazbMmhj$mCxiHsGheClOS`kT^^|pT$`0$wYDFC zE0gk==nRK<8~7Jb=esLpjj_v%traay=L(`~>xa&EXz4mi5Wc|E$Cm`D4h@65X*tDE z7-KrHi80?k^Y4?2Ha!=YVoDGvFR~~idxANdY__dry1!w$9hGi}X%3o%iXH5kZxkmA z#k-}|Pi$x}y@517SiK_h$6AsC#AuolB;12LEW4e=DkLI{wLLCdw|bbc+Qud+T*~Nz zlSSpqVriYpV3y1+*kbR0osxdC{YO7o0-bDd@C*Q3*+cwXWg`z8UC|sB-U<(d&srgffk^{F|4i}{~UPh zb~`k7PV1SJ3Af$P67Zz=sDJ+3Q~VR7!mOWf{6;80B8FxvcweP1%vmW~3cFG-o4WX< z7%kyElpR9^z>zypT9*ZMPS4>OVhR>Wg#D>xn;=n73n%Vj#*L#@p{l)R4c5v~Txvev zR8H1PM2I|ug*D=|IeED2?Vw+-MB$#i+$vn?Yndbcx#&Ct58LA9l&Z6NWyi4b3CHYK-!r4)+Rff@iJKY;nLg|RNlms)B;*S8G95ZLzvo%b8LmVO$MH) zko*VhL39}@;4_eozBD=+YD(lMBK3G z58^8#<)IP>sz=jdGDNVp2rY4>lQWJ4;qs>i_?OMllI&xS)Uyo4G4>>r%Mpd!3RaVA zHw^@)UYw9y0)^13YFJT1E<9fopH=V${n(OJON1#^7r}mVBExs^{Dy0Ro2cvhhTRAN z()w*TlwQdRuhBT+W3B>cWX9P7PF^WL_ORu;u5EF;UA6mQ8%Rs!e>#Ow>AdO8$q73B z0b_@3fTv;oLfTc~eEr84%Sd zJ3e14E6Ho+M30sLu=lV5M|kS_)gbKWV{<`$yvc2>HDpLT+XB0#lAx4IR<^+(v6+jA zGr7Ntp$Vu{0xlb#&*%w%_&hsK!;pJI_$P5eVo%#zRjn+#JtQ(?Uk!Y&GX+vW>1;Z_ zVCQEvpGW`h9=8h$D{&2GZ#R0+e8ex&+<t=MS8ll zZFEcJOd7n}3&5O73~>w>cPzj!-rZcybDV<1gSs1ICExxk&~_D$DVU4W&CIvF!LVz8 z9;*GYEDI~|jejC4P2J$fPvDFbOJZ>cqF^R~o8qqD$3)ABc&(H~I=k3>9!ULlnSgnR z>K&iq7SxZDp(_FXd3HL7oUkd9RhC0v>ptP8ySj?u!%Z@@Zg(f-#7s&LdmshU!|hFJ z2|AYKw81EiPSM(vzQkv^uF1VW1%vr7WOQ|Q5y{zWdhDHmh4{%R;W{lT8JXg*Xe~+5 zK(Gb0O|>j+g?d~|yWnj*tA0)McgfVtAFCJ=nW3|PGT*knFVP%R6ot19Fq;g8K;@>V z;%i26u|=}j+}Rz`)~=nJh|Gz*eWzvrgLvFQM)G_Aa(5=wt zbBgBGBV;pGT~IJYe31b_teIh;HU90xi6N3^k%aWRk8b`@F2Du~cpAmT5hwf{sPtsv zb(V|R$JDtx$0FlYGD_N8g}eE(T~}^)dSP4VdzMGH(s&%kgt|c?z@!yeT}pf!dF|bJ z)z^D_sB;w#J%xn+@|(4CXt~1nZ^hU@8x+l-@uk|K)%zo2<%L&OyBX|oh!t#RK<_s& zcLcnC^c<=LjI++CyQcdadgA;+y-vi;a;T_ChRBBc$|20x(E%uq(?p;;jMmt#&uRni zrUf*3VbTWkZo_W=XZ|BFvOHd0mbHi4OZt9$Tv@C*;V)`m)PvKtpAXonD5K}=NAooa-{A0#^!76O2rgrk#)wcfz6tV zu{=-Hyxql?hlF|XpvU&5qJPBtyF4!%&C)VcyF?*6ZF+o>gVmcjA(H7dRc?QF(r;ga z$iWG9xlTz1$iGA#9E%qzR4doh*3_|O;|%Ef>rdKr96y92Nwy4uzkt zvjQ1jciC*+_sm6e-4=`5_H4YS*Ww%LmDuIa`rHhe*(QTGO4&iv@eRnKRSqpF+k6e* zN3l!}ebMd7)6~_KGjWq3V6|Ez8vOm(H=69lXm6EL%T@bES%YwsQ&V{W!weU?TYrqn zd8P@4KQ;kM4ZL0}+mJ-#X?d{LKr13k8T^W#LtpWa+w!p&eq-P-FEK?e8k86t6i(^eS6ZPtg*3f;u?G0b91YT3dpSmDQKHBS1P4?GGEJm+_1g!x3n&c_fUlZI>ETT znx&VV;_ohX>ghY&6MFM>z+k)>gnk#m5B2?>9CiVNawP}iKw6CGwGe%#UPriUXegr9 z+PA^{Hd|BDgYwx^EX)2N-9mNQU#pk3uI`h%msF*aQOb?&ZkOzscNnT2I(@WsE+*zO zn*)A>^Bpe&Qc&&kPMx*kYBAf@R6Pw$;kpv-amwe;YKI2TkpMd@6)b4yoB7hRyZ^kX z&Ow^$(zIC2z*aWJ&hmo?UYIiqv5fKwlj%w5x0OwQ113Z6gIybhA(vW}4;X-y9W9kh^o2mBhq5JGoZ zdOGNuKGzNN7zE9z@zhU)v;lu$R(gtziY~+AdiU(7!McSIK2<|RI#x<_VPUAq;zmjO zxFWu**c0WJrOoER-o4)ESZ>NbRn!(X&Yr#Y)?y$}FOyPv@Mt@`jL-)Z<99GZUsh=7 z->1n@FKw!`#-CIcRd`XWRq%_}m>_WH%_;sHD(jBB86IPAF%_VyX|6M#geir-sl!3qB%p+PW4f0mJ@5cMkUoHWjid>3+Ew%P|)faoQ zwjOc{2tv~LEQH-8z7PHOE=ntds2Jd1WCApG7)nkQv}q^8;d{U>g!J@%6AC;iAJ|Xp zmND=eZYQRK5PsUQ(;MCx6=^4{yv05nY?2_VhZLA15~IRKY;d))RH$9wD$p7!0#x8j z-f|pk?M<=!d`|N(2PVv3+8@HN~EIc`HApU&T4DJV(B4l z6n8<r0VpZ zPF{aM>_C}#$-0C;O`BuzjE_1*H>?v=L|fw2Od>o#YEQjIq}K7hb*uS%6&|@KeR*`}CKc0r?4RSChW3u_Dnv`$WFL_88(2B14XX zt^y4~(EcuTHbFH2=!4;_R-Qd)Kk{4l@&WWA8Y0@;iAX5}dua{FNx=|kXLjLdG*hvYQYduCra@(E;__B}zuB7G8 z=LzgmdHCV&J0oyxxW;4)U8?n zjrejud|y^z=pU!af^6S8AlFY5YGtAkDstE^v}>`ut=|hv#d|DxTB_RVS@RT=#Za{{dP6fZ%Xq)J zHBjz9Wq*fge{Hcv^H%7y1~lm7l*tfjO|>(b;6%MVq^U0VJbwdel83vP>HCfqola%*VWmj^_&t1`G2BfJ50L%fvOEN>j#f z&b6TlhuA`v!@jL$tl)`?0f`BT%pt6u8vE5=DB>P$2|KC4hh?3B>Xqq+dNdr8WJk0k zGttX6{!#>ynfnB6oNqQ9-ZCxIs3|D@Tbn9>wtHYZU$zCQiiUB^3!o!CT)i71o!k@Q zh4u8-m4kGpri&mmmxMzi7WVDsJ9Bh-fRCqN%}(DnKb~P$J=UDoCb6(}3ci-gF0Cp$ zJ=8L9E<>GcL6^P!(y?{IJ4=SnDL+JYBLS3g&s@UGprciY_J94R{TwvwUNr*6H6S!v z0A6k~_ya_m%s~&I#RJo_J|zpbyV{G9s&)@ys2i{pLzxFC8px!B^ z>ic?60+x~IHS-=mw1B7)1&&h3Y)JjAXgFheZP~eyR~X5cQee>#%i%pAkZLmbWabS@ z*?7a-He(4`vhYpOqXq?@nCC%P+!F^?h;~lgC3v98ikP_N$S2>E)q>s6@~SghY8dCi zgFZ9fiBUDWqikX7r;WbGev}XX33xzPDN8?K2g&f-K2{Sl8CXJjdSS(E<(BN07lSVZ z46coU+fcI8ff715qKkK%z|8o>mk5ALg%Z&|ngP={lSQtbvC~KK(}+umk|>H@bsc#Z zx1CiTM5S}FvwMC_N>cb}hVDl57nc;U!2YY_fBn>s%Hg{_m(b3{VU(S}i2Ah1ipnzP z`Od=752>O_0uwX@z!@jRuB=7(6GQdLTZT{Q_mHilIQ;{-7!{)DKqk?h4cZp4o~XPynJESl)xZB-WP}12=_Yt zpNtaGOWq5muNz(7`LmA6&^mGjy{J$xs4WF_>r9s(r*{T}d!^*SKGm={t<5Z}p7QaY zdB)ODnuG;_R*|D&wsFV890m({4hIvi02E&t!=dT+8S`iTOkC33#wWLsHdwMra~5tGg>O6wQ2TfMkgPq>|kZ_N-LTNadf5>P7Z8RTyqt$(jUqEL26zw6Pxx-si#j< z(sZunw1nBsJ!!$k5P6&BvOpxD%-tJRFsX90s9IV6$XOF0iZmR*)XCZWB|8Rd&(Hy~ zvBR{m9++Bw>q_NZtT)Qj%ggXIdCwP;eOtXyW&F ztsglwgJZy(!OpSF^0#LV8>QC*(+Ls>EQ|206fhx?G{vk>$vh zrHXl4BY`}Sb-e-S8fxh&5!juIrj;jHO9)w7>eR6)eBXu8-$9UACbKddakBHi6>V^# zB0#uj>x)irr`q=JR1pe)Y6*aCa0#x+40wOdiB>zlz_KNZP=A@HOIM#4!+@@RD3kaN z$71^HzCk)o>x`vkcKFPc2XG_o#}PtNVSA2hQh22f?jG3K_T;vF&I4G2#!Mu>uk-yY zI~_NB+?&e(M3bKc4zMPlXGi|M0!mk=Uf`;6eSX7Y4;2~@{a#k4yl4?!*5A@1_R z;a3aP%cb8kJ3Mm3;dRBAFS7w6GdXst`0nV$YH=qZ(#7>UhH-o}_SFMEKehd;u0LZ? z@9d7%z*Mm^=aWB~NiZSwZmFtBCYVgohtW5iA*crlZp-q655HMnmH3vYsHrB6OHoRa zq9>nJHb`L~_5h0W&#cG%#8iADRk7pC%Lo||t0^Q_+HDhTD|`vXwdWiNf{VDA}gJ{&P!AP*`HD{ zCKtQ^cPv)SZQHc(HVf=nKwB}~VJV=!x!(lzoP(iu{*(JA=HxGxI(kihO+`q+%!1_z zx2gwX2R~~$^Ma#C%jZcg*FbIji}*YFrHXB}JHh2b6+sLpAf~)Uc7MnLWvQPVX|%Kr zysmC(!UI%(t^Aq^?J4%iqzEFii9PW$QWAH!C^LD7Fl^6 zLG#3htwbWOwaU(WmW8IGr-rz!>pvHJUjly$sZCfL^OojgDj1zYQEMh^pi5Ix-(}!O zo}QD@2v!YlnFNVcZl!iI7$8Pvf-T0T4r%^hTb#?BM7|ynX{aN76w@Z$Bm1o^#In?d zR7*lDv}<(4^iP*|xAMlr+$4th5Bq-y-MIYi()%F`-#Td7=zf4xXe3@=rNpir;cfsR)FFo z+9cOvrKvmiSihS@ykQk9uiQZ#62}w2P@M!sJW6JlAFIm3dYbd<%Ec`XyZ!08U@7gO zmF%=}*Nn}Ktt+3`qV}BsRDn=#N^3U=k-|CJN4dh^OjKp!+Y@P<$i8Xvm(fCB7BVIJ z&AZwbN*EO0``E_`^CI6YjkG*xT4p0_;1O&+N^8+-Pv#Xwl7kjj+4)}J@$GCYV`ZJ3 zw443vPB4Vy3MTQVC_Or(sMhYq(8@lf#JNlWT^MgR?{EF$ec?Mlpc9aa)+CwM{Mb}Y zN+w<=!yO}x(&H*~=>L*$Ijr3%m0s#M_S@zJ=A<{YEOs7bvDhA0H&0MgAkuVemVFYX zuYSzQUmkWTCXKnKDGXXaJ7;()a@58;P+kkxkp(8cG47MzOAGnL!oe29I>#3bCl6XY zq_+61#FtQlv-;&GDS*-p*!*h6vslkixu4W5aQZCMWrPjs6iONP9K?J}WW1pI0i?6PK7Z)&x{UO;> zv2|Bgf7W|x*HV8#(YTP^7`U%OpXguV{#c`1!}4as?&B<_a{83CMI>?zJL+CqMnZTS zwq?A$SLQKktSgq2!i{L!y+(TO>?<&aB-VDq?I=^d+;@%J!v zZX$KhC5`=de-Ahv(sp7$rJJUz;q|f`TGI!cyW*C7!2>p)lt6-%&b>}AuqmWAaz;G0 zs6M;?E#+YN3#4x)eR^kp{*3cdZ@9DX7{s8{%wwO6mZob@ROTs2Wz$Z@!>Ico4KZwBoDGExGq9^*U|2n z9Y-T>wemimmQcellxn)?yKyHznK@-}T)3w$MW4FBkw_*X(Fu(ywNM1%LCM0cR1w>I zAFw^rHMS*=Ns$a+Cw7B*2N-&^lZbq_`o-l>_RpJQuTfGVHko`BT>7!XRDIXH_-wPi z{!g~CKhnLcVcVM;i>^?paPL-ofj0gtf!cSBvu3MB+J#sl%N(I;O_x~2Rp9XWw)cF9k# z3r3Ei=VXI%(6Zth{t^=fpGvp<;&r>NM*p#;(u6`C9QkR+`A3*`@h+3X+)dlA$+oj? z+b8V!odvxVxa2GiA9U;Mt~)8Bt>IQHfr<&&JW)~>a%eZtct~0Pw}^V!hh6dkfqQ@B z5367JH`~qqM8SaL?JFeRy0Is@EaQd-?IuEMbI6nLW%gVbpTBdtHTJ2phc>iTp|jYw zRZi&~R2zd+iw)2aP-wwf$0QWQtxycn-&M*KEX!ti0&IY1d~gkB+jyK;w8xpcJ{Y?8 zI5(N%$7JsI_%r$D<>u&m9F*?#xNtaVdW&$+=s@Tr4Q-)VuF5aM8t;PAiZ2b{le?om;t050;iL|-L5E4%Qtwpz!v6Tl z&(X^8AWx62JdL-Ni?UeALre)^}y z`0|F2C(wMnQ7WSD((2EQq_<&q^0xX{Sm_#zol5`PvHX)pFMwKQ%8V3IPc?4XxTej^32|f&M;=STuv5S}FlQd9!qKoBisN3=dZ7J>UnXus$ z!*|rQ;$_ltwX`hgN}LM*uacB6yPA(ccs4EjzfX~I@k)--H3TfN>%+Z5>Lh#WcoBBu z11lOGXLWUkM}g(}dDlL^MLoYravQjlG}%qJ6nl!jb!UjWXrr1d)=KtkhZS_uETAN7 zV!u!2>yzs{R?fu+&?&I^x!(4l-(Mm4>Q$eRAhfgaf=1Nq`q4?Jk7MqX<{lke6@r??-jU2d|FiT>e|>c5T}rG`2yg+wY@&MGoCVq&`DO1`bU{Nyx|llI>Bqn8J%?)j56cvfu2T^7gn_1Z#I{ zy5T%m{YdYy^q0K?PqtoHkRGI5vf$pMIJj5iiSw@9uyDd~pWFNElcLIHwu2I@(de$k z_bXHOJ1`qyzS*{DkG4nS)j6GfQ_?pbk1>q7E{BNe^vY5lhqZh+BB|Xz_{vEI)bkcA zEd?1eTkA;;%;@$R!GaNFGV>rg^4ZpB9*b$Cit)L*qjp01g?*II=~Yx+LKXd0pSClP`| z4dgf?QN_j!&yvJh+81%(9(0ujG5Ol*$~PxTYUoxmPo8eykJ_;1{|)UvKD#Y8oF|S7gFKiU?}X#xKKz**$I8MM7frB zYraNf@s-&oeu~5FH)Rb$4z6|U7F(o+62v68Mc~g+@?i&XW*#-0>^Mq7d0((%?}zQ- zo<_CUYL<}DVI*XtB(16Gi!fBA3wPp0e0WZpyJy%?UHfk3KG{s$4| zm@I9`q#n+0(}KT75;D}W#g>#nca?U*i+Ei)0Kdu3JGA4I@Tdw?KZ+35*<=r`7NY87i1h| zYJu(g-FbV~fZY1d8Q)dEPdfo~1DvWi{ zg3+zaFw*p?NDZY0tKn0V&f)PI)uYE}7Dui2Pq0D?zH`*u`a#&MQz|Z*ZGiuJJfj<+ zPanru-YYQYC;EBD(bUMEvU;R9;Q`vQPc3b`*ycs za2*{F<(aIz1yfoJR5OfrIDGON1%e$D(-6FYpFT{t3NCJ1qzNSZWdHnHqwm`eN-X7J zCvzH#)O3_IQO@^CQB1!5A~3Pf*bg_6kvqu-23CC!LL-SwIdb8*yfh46FTF!vw(; ztAVW(939PVyPir|_@XkTGWD7b$dHPzysoyj1<%r5aQ z92+*w)xA7^OSfP?zssdcwFov>KNTpw?o#`?j&EV|^W7P3yXI0infN7^o}nZLv{%oM^gV?Z@1du49v^_h4ZO?h{n zX)RQX_iP*Ic;TBTutN76x*j=S8+ai6nT80@Z>0+f>1ZPjM6vXqkCjX&>vsMWG(q6| z9FjQK09^fuh-E%FF$_=fPL_aAfAPh=pMEjqk$4DKEFV5Y@`SXCd`rrEuAv4qZ3${Z zwDkGfMRCecr?#FtRh;?3iGc^w{f2Ji7ys~iF7(PRIC^B4ZmV45a$8MA`=iF&94nIU zDFZYrty=WhNkCy6-l`N^cskyd=zfY|W#ue~(sC0`mMl(d3MGV{<{T7$FSP6U#PE0! zGP<27{YIxrOoEW*GC4U9m#&HVfa)$%OwZbYr&uL$VAx}4t{+>C_F}P2g4_0vKKA^2 zoHvcgu8x$%&QPPi&91ofVH;BV2X_r}uAWwHN<}dCXQ{>*evAW@byN#|E`k+gi=j+rYQ5h!R|hL zEmUP^THsueo}OO+$84Xfsc>#9>xUoyfmv<~0F}1Or-yYvA6>ahm?-o80CsehrC;|Z6ujBw^<{gZt)sz`le1f+5tvpx-ohY)ZfDY z&8j;<0!($#W74I;`&@g@275G+$FCaRGb;dY4h#vKaA%(k^Y^YyT$k0t z&iyo)Ah$IMFe*dO+PjY-W95*pdmdZqZ*FR6gbBVgb+Q*qzNuK`TdaI=cdbCZDp$Yu z^3Nj7J>1Z(+K`jkEO~d&b8+&BaI0FI{%PUN@qCe1HAhRi3F|(hi{WM-;09hn&nnL_ zcNaIT!)=Qr9N)8?e%F`8Innr^-rtE=dWYrS6af%P$W&xt7d;MjpfCY}JFAanyz|z- zU)idD*$La|eX%50bN+SWst^AX5NQpv94t_l1btt_33TgS>&y!xbJG~}bd4tx_K-Q` zX?#rc;YOO$G)8J8f_wW_s+Z_tfu4gP4+f4weXgT@?{}j#a_c0o#YQt6(Jv3X#(p}; z?d}mS@|;dxp(ekS#m5^Ullx{ysMaU6ZK*1J3$)T40N)Mqnm7xn^<{X8Ae$HjYS(W(FNnwA^ z#`QGL1PvLwws@VgK_A13KAU$eoO6De+vx6eFwf!wJui6Es+*<}Z>)(U$kn!W9A(1d za^Tz|e~Kujukxe&{lVpmRLScpg=G?@IOA@@2dmVq3?7T-VEQLmG?1NU63yo%#6ENXDM{A~ zr~xdDby8sY#q0oS(;G^=e#i5Ua^R4hKW2yxCS>v?%u+!G#M_;5&&A?v+Y@uUio$4+ zuW2C6A8c+WE@19X8-G7-mMq`h@oZX%>@8}y5Lwn)Hm9#yk{i_r)bjeg2ABVcx10qt za&BP2`u$mI-1ziB>v;b{;=3Iz53)ZPQ&8e6(;{`ghSVOzxGI|F-ywgM1w)kY$n8pJ z!05ci_Bo4a*`ltje(~XFoQ!C0)ft!;7S51VwbN5{R;!X*#~eo6br@xNk$>13*SOK# zY;-v&iNVf1K&CDCZL6@Ll*Hgs23}it%hY==ikD<7o^supXys!=>a;rdC${>Uqv-KJ zi*3El3Jfkx`1%u|j)67Iw5$shP&ujO*=heXvd%KX`aiK_CsLTTx#quCs#bY4-q<36 z%&kF7vW|q^WpRAPUhhD~`Se7U_pvXG`NiR`$SP4~OBrbrx%KpFM_k9A)4L^Nb<{QC zT1>@)kEt*V=5&ml0zW)xtl&v-I`9Vs96M5!e}doR3~4_BUQ46&I9f~M*q4;_f9(?$ z&Dr6~GC8B0<(+gK3&r$Ej>yH?#nR{k`8gG1Q-hkE(*_%cVMjRl^*`s&UT*W1nV4@7 z*M)1Rn|biIenOl<5ExEB^`DS9Hz1hUNdcX(TVMOY%K!B1<|U!-#` zeBqh(7b~Yrd9kdzezFRm4Uih$I-w#Z4URbIn)j0by11dINpVWlh}d^1-d z1iJg1n%0_8=&6e3 zh{2mp17yYUVm+jRcf1k>%*4R3J%2zmj&^Os|;d6D_o2!gjz9T<15;eYQKJ^+yyVwC1{HHm&$;Y{B&7*;i(;4P~S|XkaWcZ=eZU~eD&6&>WwH& z?Hku^7HSr)bQ6fcIq{~$O-Ii~ykp<5jCJ=|fF|lw?d=p0OGr-_7)&yZz^id@3p^#F$4(D_~7=j2_8ZvPb@U zPUh(+fxr&#l)2>!b}e-7M$uHfk59|WaP%zCYB5edffmo3mHNoPPX+nLfd{6kn=PDl zY5wIT4H^lMLXc*Z4ygxsd+O*pxS8~ofR)MI4l@~#;)wW2^dXgf)Ei984TLy#W{%bG z2tNxD;eH}?yL#yn(H@NgC(DrThBlt&DFq`VPuQ<;7IvwA@ecmd68!9`1D7nFFfU9K zJW!Yx-B6jQV^{M?rUfZmtL;1N`9Z|8Dy%hEaDu1p&J3rdpKIyZPhBuO$us;vF#ErP zkn^73!b@Ayf&!`)v-G5-mOP5#eOu7DHWrf(fTA37~vfmyW_q%1b50&e@&v6B9$R0t<{#p#1jQ5TmgXT=WCn9SR4Qal> zMv2(*FeZk4(8974d0N508}6Dl0lS>vIyw?$w#5~+6rX?ybJs(~Ne_b!^lc}4iQ->s z$`T!THY?+#TSV+`FR<9FZMKNqm!pcft^J?7gwZn;DlV2aZyq;6bJ@&a{B^k)&1I4l z2O+}sre#vPog@o$0&)AHuanM%+x&ue%{2nLhh!$yJNuv;%(nY)p`C(}{(oskae zW9uw0o8&(pvW(O}*Kvvg6XUg41h3iqyk6t^4zXUD2KULr=%tpDhcSxlE#ft)x2Bje zd|4@p%QfqxzKERaM*(s26(4Y@?FwoSG(VW#L+E$jX3@+jE5jn42#%n@?II4zr*$|S z&f(S#X+nSh6X0BU58z16khHAY(1K4_)ayO5e`yofIo?sVx$k;(?P z@eT|yOt6K_`tfKaJvst${aldfae%!3fyRPAFnm@&de&*seGow)M-mSKJ701h=t=5H7ooCScEP)HxewCCQ~D>Y4;ggd`085rT76|e652zJV4wn zCDdi}SC)O8{84cPBFNKu@{lHumuq7O);#|;H*Mp1A(%EgoxsT4OFvRg=;@C99v@mW zS_;Cdu7>-;S4NAl^y@81-C;aGSi(WM*%or8?bHwVVpFHg{!Qc0uK%fg%whl zu+5{Aljd>EIsn;^(9;8d4P_0;+b7}AI|gecix22%$yp~$I3YVaIwZBh5*)TV49<7& zhRH2|WkmWA?W*>6w_p_&QktNbtGWc>0L@a!V0qCK0&vz{x}ZT}#uGwibo%xWp1b(! zl~w@_h>%J7wUTxTf*(4~d-A>H=gO2hy{8NMs1;@%Py@n$rG=+xBK$O)h+yDaCTM_J z-{g$>`p3JGB5-_XMT+*K38TTh_f+3DVFI2&*~=D*szJm-@FHi`=$0JtI~i>73ThBds?zA#Mv`YNh#S|-~ta$JO1Ua4{KWwO|w`Zz&k0W3Z|GksU@>9 zigi@yLvsrwr^L=aP071`@pr+rqL33st7a}&fS0N`Kt;1Q#7a#jt%cWWw

38N0i! zJLCB4Y*SMVFO-WJT__iDwH|Suy=ywF?Bvv27UEHA_jEF0*-CQ#_V658zT`vPt5Rg2 zrjfC&Z3}nof;A#OUw?M)(Uuv!zaosNDro#U1-I%)m^1n4cA9rP+4mftQpH~&;=8%- zPzN{c%Kyd^EeFeaWp1(qs>_I#}ckWf3oa=SkkB5fy zc!NCk-Ey2n%FE)~+VcXh87+G*WP@cMT<(%Ex-352VMr|rTb(}ss5?G5mbi9XPvL0; zD1WplaUE2TT6ZJZWci{az?lADwcyqHVB#DWuN6o}OHP*x2Vl%^95OS!0RJZA zOPZH_;?W$<;W6J|v@yv|o$WYO*_Rf$>uvAgP`bG~MZaNGs=BC126GT$daq-X%W{LT zKYdE=mQ$o$Mc~R1yT*}Had(|WcvJrFk4)zGk<{N>;U6r$IQ=)4n?cP9KJd3#;09-B zMJ71pd~U4EJK~Y;0i=@jMl6i^U2HP=Swb6d+~q1R?#)#+PS?;$2g>&E(0Hb4#zh(? zSj@TbTsVR^qr7`}$4RIpaQ$97q~CjRcQ(R!LD7Dy!u5`PEmMc!X3>I@_@jor#e%=?q|E;=BIo-k%=c_rOsq1%4_?g(8r@q8Ge!EFS0Ow}MxJX& z^0lA~2~T^hsefjdCK@Tsmd`5^;*hIvdjblxv%Ig=Zu}TtJzgw~Nng)GHjICmwJ(HZ zrz(4Cw2)%qu|(QFjf6gXVRO|getIoqx$Cl zVsuxD89LO-0i(3$g}}MZgLd}z;}&QP2IFp5}I^Br0N2$e;*vdF|dO24w!%_jKPoj+-H7+E%i+8Ku+Jy!2;eSxexsGz?- zvOh4_1mC|5xPvIg211Ksyv;l-{yBbs4oCEq;O*AVl7C4x=}L=^pmi)y9it6vFK>J ztsk*k4#4r&;t*VEv9JhisWWhQ)uUYu`vZo9qL_8_Q3U_>Ark_PgDJ|dLL*&X4(nO0 zL9GaFw+9MFia78S;@ASyWP2TKtb-`l>uH7uf~uHw8_}Y3xGsVo6fMUhY{tHRUv;Z; zQO=k2V^}#AJ0-tfA^F^M>}v@lbFr>=QGs#&h!03cMn)q6C>f;}L?8dCMALM%h%0Oj zbS6=i#X|T$8$kHK$qWF;!-~2>T>iP^qL{Bvyt)$HI-Q$Kf!KN0I`hJ{9$^bF(lW?9 z`Xt;Ohy%Hp_JoQre{{?Bx87+5{*s;q4mq165$J$Zvoz=ch6LA%#@Vfy+k_m1DX%bWTM_+8d)}`G z65l2b&v%0x&!W0~-#S=*Ca-yvkV)GQ$Us||Q{8$_U}-77gXbz+mPD@}Zam$b2_`N( z#QL$>pbxBXpjY~X`0S_Ow)F2r&UNk9Q4HMN3BbawsqH}hW`*|}o13Srq3fSbgKkit z6!3CeUmU{p^qie?8Z5uA$#(CMbMcav|EvxLz6oH;sL~ML1j|Fq0P30_sSkBV?PkvL z^=5;I`-r^SV0^Yl@|fQ92J!J%mK&WwXg|`D-+bgJFXFYm+&r$h_@HukV?1V|F8D~r z$Ff0}HmREaRC~M5pF76DgEuL8tf2zQGj-BQ7o>V_MYkLJ zFFm!D0igJsWEY`Y6T~ENwSjKAJ5_0s4(P>L8~rP!ii_#SD>XT7O=!O*TgwYsSIi?O z;uYQr4uChL2jJ_B3CpFy^CdWyY@1YNqx3xJN{e_;E8YF1bDa-zAkX|3~NE0Gh- zQ26$T;k6^Bg9@pk1$T{$>{ncDROz>C1ETHfV;cN1-lwXY=aVdJr8ON}TvaBw<_c2_ zqo;V*Z6{)_zhDC#U#w?H)0}loGX*@tVUY_md!zhYQFBxM`F!0kpLKKHcW16|`0KCV z9l<57k=|S#=eqeKtLCv-=;zPhU?`{}`bT!@4}Ju`%qixGK4R8EDw&M2t5uHSjDqGm z!5-xw!sG)-#u>aRyH!9Qt_=*V9)q!7xR-2;Wkecwh@D0{czSwP=aqmjIyyO(4G#~$ zP15y-aSh^dtci&eJy~kBMt1QNh9!?Da9EnAQFq^Du}pvMO-B)#1&)r z)t4@{m<5w&I&X0zDeL5Z8*pCfFqn%|$)WNVaI%d+MtsEHOqQI-v;lTNFG6^2Vz${5 zmkmlX^afD=jlDOl*`}YzA7V3HiGs4E(p7|3n!XXi*4B245t${O)vJb4vB#oy#U0Qw znQN6vG+WzI8XkM?8sdC3Oc)CW=xG`DU!I8I?#mf^p3HT4Y))0x_oSqxF}-jB0Rd6r z+XAcn&r7qjn<^_S)o;RSL>A`fOY`!20+paneXFRc@9(Ybs!~wQrTH0#OU43>Jow($ zZ(%sy053T^S1xSqN%2z`TISN^d_5ctr+4mEIU4dmEiWqS?wuTMh;Uu%)Gn1e#pD+J zFR^n1Ky`ri03PU^>#pF2Gm3WWrw=S@9)5YwRN9+=V$wN1rbln18vHJM?T5W^jx1!? zXyVJb6`4qJe_Rvp+F}&|Jg{ZpSv`glWa->{S?^-cHRujJf#C4-MwneQwl4>v!6SD{ey zeGee~ii2g|tCIt9d^+3Kv%`W7;>yX^vte(Z|Lem(b*oPQN&y@(eKLVDGsj4&{%cK6 zW>}&o9sn4C7)P)LIog*Dqjp>P(lb|T);T)j!@Ng}TEi!LW_^r`({kvEC<6uCDX4VI z+QWrYL~r_=zTL(Sf)O$c`$@{U=Z5nMqkpV%A&PFVmK4CCXz`s!mguAAc6{N?+^l>_ zo~a6pbU!FC6lUI|LU;eKou7T1+<}K|#M0d%hBJl>DKTpg3O=N)sv{-3Dzkc5*KZ2l z^gcbUp8fz*>yM?X3YfTI|D02b8Gge=@BWIhhx}(-#$OQ~{Ne10P^Z30APtSzjXYcs z2r@WaM$_LX{`%0^BLa=4UUH-OQWdZ>gis!{tDKhHgIv?WpBYU}tPxt%$Q^VAe`KJJ>`nMU8f~C%U0{$3I8YBh0;KaC6v(_+DMB zDm>p+XXukTcwjYUnC0%7B`LR7GXvkQfOSqlN*9%A-&Zj|0l=M}k0Si*#?q8%s9Fn1?S7yuS3<+* z?D=5@*%y5-%iL^G@i6<{Uj-bdPV@N*D^RLO>ghEG(g;l7xXN;DIU(=uDIYyTe0=;} zsK7Pxd6`qB(=lxoX^)b#Oga+;jq@ujrFYe2Y++4RUds=YMQt>PH?!k89E?QnmG!O{ zb-DB4^K*0gW_DCau=qC(o&6T@c=CQk{tYD68Jz86yvBpERnPAcGSs#)541?)ki}TR z{dz4yOA83AZXL7n_Ngx+Qm{eU#89sbHFRxRHDDtHGdLp5*6z}#w2)iyZfte-!<;v= zdk-h#%^g8mr5ED6$9jf_rgE3hb_PG(|5R<1;<6!n1^w$E0es!S=<*`3%u~r6V*yrGAJQ=DGaI1NdEqek`AmDkDX6;xR5Ow^X|%e)B>n zPEmYDhcoU|1EKab7*MUs9Cx0#ldsQEBd{@L4*=oy%lLyi7V;YdUg3HBtL#9s;x(TRNl8ALtddFY-2^psYTfpvPt?H z@tcY_KaS28W*vJmOVHTa#9*;3LpRO=P)}-Ls}$8>fyqzekL~j`;)3u&xZyc*_A`r% z(csfF=7+h*A8Dn@UXTkY-g^_fV@<~?`vi}Kh)!H7gBh1i_ApmhISxt7IFH>aHmLr% z+!`+c0lGV9Gm#Og3txRTkL;XOY5Q3jQQTfQl<)c%#~-9{;45GNw$lxwdLd>bmOHBc ztu?Pvj}*8&r}oh~Ti#x($5vsx(ahJr;f#zo|9Q`q50y`!3oZ1wNqHD3*@l_%%=xg{ z&e%1hVRBTCXl}9cG7)q8l=WnGyM??Y1E>26n? z{!wy0ZU?LvA=@2L|eZ&x4LS{Kjq%NZ(QpYGi(+Rop7yYOlSTlxK~(UHvUmy-U!p;QwJ z&=+s7aoX!;cXVjt5%yRH#63|UvRRKxSNSM4*hUw)s&56u%#&?r!V0c#|HaA4lE*lC zWsA1+!yuqq@#mcGqi4P<&gI3+pt;ebsdR}0(^(`x{s*u@eX&lK|*-MT+hBY`- zIcAayN?IIZgV#ygX{Iq!rcjhJf&y~F4u68dV9QI${0!aNqp*)`1t=)j{jB)2rLzz> zH@6ft(9u+h9(h<}1}3zE)RT>tu@beMfNF1SY;5*)0qBD4^v%4pY!QTf4evabW3(U{ zkE^@>SGOJ-KS{ajyYbT!Ut$x}*Vkul=4oYRRcthDZ^#Yb>iTdSY-o6)>_WkCerNb_ zdVT48?B)rtGX+%sKfb+@jXd+zCxvQ1dS6{UELT%sYdoEOwdV03Ct53Wm%(d=C{-)* z79Qg)gaqG9f!$OYwvWZc?*0@*HT#&I9}D96VtAW$Rp`kZ=`pl#T^6#kyuFzpplUS* zk`pG0rR$Z_n)sfT2KgFIutJ)`m-TpzmEKSUWU8a#(zd#k9OX3y7hTNQ6pGTHKKV&c z@7Wi+AO3eCTy}i!H?HTM)Y&Uee(>xvKs#7b2{v=zfzs>0DuPPWCuV}a+A0Jc$MvlS zPS>|kNTU8ycLSyWYSQzqD~0O1m~k`o=Uy3qh8w^aPUR;h%Mub25+qMx&|)8ofG(jp zVAVS)tn7LhO04K3s?QAI5gOJHOk;)6oH?gSP_MNTu=EK6|)1UxxRfy>3zzO_X zfKsb`wR}w?`M?2j_~nBz06GZ9II z6*rW+dwATbZ@@?7N>u*tBuDb=KB>g!qfomJRLWNXx4emno=1J8P$e(%_JN$9J|A;w1@z8Uyoy;oVEph5;E1J~G3oS_~>h939ZF zOq4V5>i&*KI&|0?OYXQa@v>XrnSg$bG1YS`Bfc-_bi-ivHFHK=Tx8!;f4N!u#xE^} z;)1#MmYT_r_1I_rce;nu8w09r?}|}Zk+WqI5zNr334}g_Lw}j0te!Ru+$&g$C@Y&H z!nllKAM;V@f}A*UbU`zI%~fT$Dsy+cvn{R|>-6JpOvdeNJ6qvCXDE zNzB<0@LjjAyM71WQs~W=IXc0T;q(W%UYl=ag&JCHFScCdt1c~XOYs~uYf@A#OHr4m zzDqJ5dGdkjrOF z(DMOv#J9DoW#=H4b{;LX!D8n{p+h2Q`^@!6m>+2CsPu#Z>c-!i!9C_K_MTT4L>+dw zH*+^HpF4K~1!fky=ZFlFf2Nb88O5V~oJ=iIQ6M^ozg6{P@Z+TDj#JG-{YNo zy)pIq)erTry#u{9!vb6!zeH`nSlCW@^Tb;0{OtuVqY6l#nL*B8d9h1~Sg&SbGFZGH zlBn@2T3}3w>0W-&=*yFqSuUDs#lO?{E2qOSkt)lEWA^qkCAzf%CG9Cv=*OLDrVNkK z*&5!VL;UcJ{)hFomYK7@J7i+TW$LGSOH!_rKYDuT01`F@yPj`Y;xPIpY&jKew4i~_ z>Ftff=nBEKlg*~bV-z}Qz0$-XNn#Qb$yARb>~s#5b(klIKmlRJUM*2b z5LXRWCHe5rV`1blq@XeK2h3|e;q>X#%XS?!=fKxW#A_7LIa-uHh}o_O={S_MzB?_~ zx0NaWCzAeN#f9?gX3&iDIJI`4tsD$aW^2BIgM-x6ynx($0{Ga~-@C4QE2LHy$F6z@ zsPO5HUj?+aWogv9$gVWRT=jinS}V(}=vCje7#@rsF0WC?p`HpHDBIb4=HA9yl z8tBHx?s}aytiE~^z)WXftMYw3^w6K-B$p7N44!z!5v-QzhQqTN!SI2>=PFC(V;Cx+r0>|vp{N*&9 z;0!1jCg`q1u++BA^#{&cfVXd%Asq|{%fLLG_mYjWBpq}P!zteqrR%Tub!W_&K`a{-M$bgn00DrP4Zg4xIcU z3)ee$3m`@fAbADirVSy~wBPaHQ@M`+9UL6A0hg3UTGhUD?x>eaJGzMmtOa_k=)!Au zoT0!}^Kl1ku+DvL=ZE#5pE?RlL0BfQzyV8sD>q=xn0+?Wh|^AeDp2NU;CUius>7j> zP`&kbgD>xP|5c6OUJGqqMOQJh727+-8PW73_JM=FW1*6c!*)VU!Lq@sg7j>Amu}n& zct0~wldAfmx>`VnUU&)>rbn9eH5l2%{fC_`l7(*^iG zwVeJaXf~c1@buw$HTFDu@3hehq%VokHSLcbePGspItAoemni;wy1~^nLhUrKhH8Av z5J==CxS;P+Oz9UVU^~x()CY8L{XtSLpPmo^P3w9*deatdS8hqT-Ions)F{Z#mnm(~ zAn3(q=PaJ;OW0m8vD-2_X?VB;)iMpxpcphdb`v!}R3$-o!$ZxqlEzYn zJlh2F{o@(PKfJNOVd1X^w)`g*>oAQniCaP!KjoBS1t4;8^@E~wFV#--YbI~4wk>-r z13;yyqA7p-k&hAHQeo7&*zui&dUHuOrauS%9)b|NEfFoLo}($TpEG=czQ1`#`OqPt zroXE5myj5!)QTP`T153EU6`jY2N3MSRoX=UspIzjZ_}%m8md1?3wbzN5=Nb+cAznZ zj(~xKcl-U5yQ36apQ3$Edm7jtdQPL2npR&=N#K<7;fDBbRDW)Qt7_cuIp9OV4u|Cg zm7voUWv=vMA@>pa%^!?a6z@f!6WR_1hzrQTyUnh7I2Cv&Isd&?z@sBi=8gh)qD-&RuQcQ6rdye$8?@zy28YiMccPn7gNs)PM0_y1YLTe%|9S%nQn8 zLLJK=WPqQ8*K!_zKJ0;qVEX&td1}sK;bj0m$5H6vwY)rjospSRTl7lnOry+FwG|-N zB-u7Tgt_j-p=E=Vzks4Ud2OaO z8mQXT4EDaO?Gk|1rgySCQ+d3&5@0Btn@;S-6!-$#QMSdC+Ql+LfDE#FwL@kV%!A2t z7%cA&AVSR>1ytq`1;)e%#h)MSL{I$$n>+lw;tFnkd6E={pTuXI^312k7{g@9-3D$tZ%RSW?h24tco6{~bok=* zXT48lryQS3n0@VzGMvduNlw50F$x&&NbWI!BnrQGSheYUPZf@6Pd7|w;mu!rIlY6m z;gY*o-Ww?}@rdt0y$C(?GsuvdZ1=Rbqy(Mjx6&ecZv$r!M*w0uK36Aae#n!6U@`Z* zZe|IdDCL{h6&U0{BsR$Ol^kileUXlCabY1!e?w2P2wZSlp9)N%F*>~!Fq@l`4}-NA zR4rDPFGWX3OFrGmN;E456t%BY)Bez!M+D9jRe%9wkoXBP191BlZZka_?6Ht=>}0cy z=i)1kJSbF-&)wYI+=-S&NS2378u63stotGjW~j(Yd9gCxd-8%zz;?MQAn#%Z2OWxg z`8$f;K+YKN|jt6gA;G zOwHWv8n&|Aa{zK~MYlD8SnbjfBf!F?B#FFg)q!*^-@RkA+P0tb%nSDL}w7rMH=Pp444s84TF z+hqo!Bjm+8<#8T^(W&!73ns_5#vz47HJP@ttk%tXR^;T=($+O6u4d_&M}S}Eeo@jkSW*1cAJ)Y;>MU@-U`zg>w#@qOMUk`h=(>k(qkl2)iQj1)t32Cf=qEhal%tmu1u?n7=Spv zQLUD835Zj#;5OkT2l^ptENZ^0z(Y9`SI|AJ zN>guXPnn7LIhu&I&)V^i%Xs^)*~{XJ!3?A}9!1HM?e_Z=MsyIB3E~zE9PdvWuvq}Q z`elonkT7B*(l!ep!Rhm@pn64B%{W*e+QDC5+CS-UA8!InFm@z}iHS|QeGo_Z6qR}Z z9I1a>FjapXRN4pCH#SP1y;EO1D#pBWQcKnSZ7|w0z$<@bv>6mAD+Kp3oH~nHwlZB8 z!6xykxp#jA*Lv+0!UzG?8-8x@57$pr-#2pK z=$1)ZSe$#Dd=A%xHabA*7Hbrh+J_+1!4B{$uY&^4`;Q_wkr#br+X#COiI73}FFtE2 zNe`1>o|=@<%O%oMu-d~Z1*T2g!}a&8#JcG_!pW*tb~C#6X?lsqe3Ap!Mix)q``&{_ zu|f(}A2|+xQIYyhcXM4s9c4N?4?VfJJ*80QK8I<9lDL^mi}ORm5oCCXDLN!VPkEwL zWElf9)_|VT0<@b1VROzL9=t9aDyeWCrso)o|IrdVj&e^6BZK7-xq9x&$;p7hsc#Q( zb&3MEq5^}WFY+5Bq5X~emSCSESV2Jt;A~k9i7A=oy;srC-)QI_UoeN-A>koFt5-1x z=5lw5otQ{8=d7JlOSsDCH8h-UOo1-WuU){&#nsavBWaND^V{|p&EFs#1;<_1b_{aP z2ga!EREea%l6)Sc9s{l0G6UWAARely@#6YG&6L?9z}`Lps68)_bNJT8RT#IIh-35z z@)~33fhmzleU5^e^cbSufj7m%jb=K1w;D3u6jh-uq7m@Ba>WUCvF%vQP0yBqYAUh2 zn-g-F;rDwd=6Z}P4_(m-IF;ac?t*)RUNFm@n|1Ntp1AC?VcC*lp%(#u7$pU1#KjcO ztxQnX)OKi}d1~E=oAQHws~nf9qQHb*BhIQh`aPJky*&cZ!k}NDj-b z#iMrBwlH@$w|+ob94FK8f*Fdy+!)Vyr7!~mCF3+THN7R1ND;*vh`xaV!xg(P0d$pS z)Mw^!5%#Ndn7p1{`Po5t&Ru!BUKd_GyAVD3Hx=>^c9-(6^X63dPZilZRtl5baJoub z_I;K5G}4YSA<+^{C#C7|J*~|woykwsl32&CYqb-LXP@$Z-QN}fKy$9P06<)3lszwa z`3Bzn*_P{Lv6EbM-oHj#k8yVm4B3!ondxJ({lolFfu8R_h;ENu8~3rRmknr2hQ#h zzmreuOrZ02a0P#x?sQ-&3%jdfgtLaVi5w(%dE6;HD7SfPyb@r3FZIQ&{vI3qFE_40 z)qU+rcgDDE)RT9*T4Q~GHllb%QS+3sTa5!3f%CALwHcf}wanRN^L-U07NjkWjd7Wv z$NqQW1~OZVS>*^#c8d-!X^x?lII zC#;k*&hX0Mu-uosfhf}&zdTd#Wu`#Gr*JM42v*7cSaH0O`9$8w=JrR9VJjj6Z~l$Y zyr0bfss8`|-n9pmsjJ+jQ&V98&-K7UF~;kH!=WXMxtJEukjQ`cjQ;3;|K-A);B^Dm z*PG#o^`rmI=loN}`qv~{S5Cp{gVlW|y!%!CzrW;v{Z)^TGOY6bch~h_F7xkSTCh|6 z_R+%*Ld<_>NdEpK|GYl_=`*OFZ=U+UT;={$kw}%DH0v`KcUf(`h z_v@O^D^o0wnG(rr`z_=D<$i+>J(NH4ukY-Cyzp0-D4lS3a(mXl!=L}dqf-cp0ce1Q{xJH*ZDu**1x&%@p}Nz<-GRTHTVDFF)=89xB0(Z zsHCzvun&X50e}n`?HDLPk?oR2tW^gV65P)+?He|N&s@n^IznM?NF)y#f3lm( zlsNFWaq*)UfUXv=n|A9UJ&9A3$z%tBdqx7hH@ZIzr9ZuO$I2=bK-&X(deE`u@Y&hf zqLX!c2V-0rN2O(CAf0J)%j?e{G+$<;_K^dlk0vfLje zp98#8|7;(0yg$N4NS$yqnnUJX^L+;bN=_b(ClZO`R!yg>0K7N6zDLGl068$0Tcem_iv~=O;I$AjEtenP&fu%POJ>UBU>`qTOuK#md!yIj z2`-^=Q$RnTN)H?kScy&V2QZb381oNbJftk1c^Z1?$5P6qEg|V1j8@lreLV=s>G}LEU@S16|wt+j?Kn`IXI6r^NzTi$|;r7(tcDkIc z$pS#>FE3z*TmsiU16c@SQy@)vUfq}r60|*XoHXd#$!uzw7rRAE-3S9ncJS8>%27&HQDx`!!d!3B*cXNgNuud0vvS57#W1 zPV&m%($en@fg1T65U%*4+uJIOKF4WHthr?YG|z*VQ?1iV-2R^3r`5(Gmx9W_$rRMX zK|9mRq#iZl_ZNHHHs2;=)&MK^If6e+fki%Z(AvAGmRKHRzynwP3>jE~**iZBIO8|i zQ5%iID-#l@<8S8AP|TN-49m}Ue=jkgI=A$48}&d4XC8cPS$8f znQ-PtK?Yl_*l@@w|l-K+?#@I<2T|#pIEM}SF4c$wZsrh{!LBX_ z`cKSP?{FVHH>ty?Lu`xH$B)Z%WiN*v2l;gYsd9EhMG4re7NCoM5w;35iq;AnBS)$x zlk6#JY%pJTQvQMo!|&e_%F^|9z|?aH2w)Gx`X)O&yLV;0ya%6PrKSj_vyZo+-ei~e zlVtfR(*^A<4+Ed{+nh*rh4f*ucz3KR3tT9F;{-LMJ@d^dYgu3_&sjEXcIKp7Ex8Xq zpPlR|0#pC9PfDhmVo@B(f9=iA4q>lmriBVMGOKMG)^ydeV&dG^;FP?6p^88DeCPG| zM;TZ)yr!e2hNPy{EP%0$?nQvLka78OuevfutAqK;MPW7bCwltkj}f^waDew+&Qly+ z!P4;W))2PjwjygcBE7=6;`?Tsja~q%c0=($)U+bLjGM*T+1$z3t(3ZHA7!O~yuoP`f@*;nJm!AgPAj?ta+OCW^7*Evd(-mXa19 zpiN0MtQeO2{@TQwtlzR_rhuqUq&Y@8+YIYA4njShrw}6OQkojfritp+N)?L{4zCj z$?0@OpS9VaM1_yOmB2C@8!`32#{5&}od%RD%seUas-a8AOT`wFyJzYm4sE5#^em^%7LdlOv5lTk-#h@hWxlui?~2g`m-aL+70>UMhtn5P++Sr=3K z@IsoJ%l?{X%<|th3;lD`G$*H?OO-npEPH2Lf(qNVk0xX3C3AikRoea^bM zy1Ldg9|x)q1emd2mJAPxARe_rWG+Yj%vEbac}4BFJ(si;CH7>0)*W#+`7|bae6{A` z57VJ(smzz%u6YfC*<_1ZwONbM2=__Kn1P(fZMZwuq=>#|g*YK$B6ZI^gjxOjzwQz= zuCJ)YOXH-QCgGGe6Fz<`xrv48PJR&`q4ZXaRGTHGpFc`6(_=mNN1A^P#Kc}m9A#}&=x~h=( zoiuI%26{9e*aAKQ8@u?O_DJZeNGJ<1!%iBF08KvlBHUGRXMcYkMGJMBjiL3*Gb*0l zbKMrKOgdd!e1uC)muXl4dgK?HC<+3fraKbIqm`Dzaz3T;$KkJ(S&m2s#gr>|rGNj( zxUr%HKVZ7)1Z3D)FZEjjr)GVhp$D$qR<;S9VtBxHzh6@&O8dnAZ&1pr115#WgiCf!6@7=ibNZ->yajm2s&=88wVsfe zRn~Zx4wc{b>LA7Y!H*6VaSy1Toa|2B9<6&v^<2!d`Oo`&MzyuRR0i7p8R1P3zQO&W zA_|y0832I%LKBQUH%>IO{ITdjv66s;zxBZtI?0k_S+G$!PdKAr`s(Ae!Anlb>G+!dt z+=*WHr{earBb^|vt(VJgMJy{uu>uS{M?ym@bS*1KmRk;ML=JZWBEC`HvP<>B@V=f;DE;(T!MF$Q~+OVv_IMN#$D zw7tJsA`6F;(=uOx@*HUv_wX9g^an5$KmtV2FR0bjrBBCLSs+hNG4Y~qw4X2k_U7ry z(H|yKQ?dCu7JvN#WmQz>8c2>)F=hg8=&1$&^%ya<(IsW{h~x+BUGVqqAhU>_C+*~M1GnFN{`by>dTj$xL#}*ac~t% z$hwkFc{V~;cC*i82-z$n$Xuf!OpE*74ciBf2vB@?x;RXI5RiE(NUI3abRxdhH*#Q7N!aj7RYi< zHV#zSm{3PqsXOoi9LxvWQ}NG9Skptl)5DOara47o?T{k2w z*1C6q2(5iI{Z{K@J=zUq{v(AGU~^CFY|dA1_JLce`I)Mkv7?hHx9azS z^i;>|A)K=ro98`D&O~nFyG2>YnX)S0Hel}Oy?KKOoNw^wTV09tGl?WLjmXjzY_sL# zp4s)$AM0rQ6$jRlF43a;$IkLd{sG67p5K7G zekY2`>*Pu)zLUM`{rHE;NR9W4By*h4&{&yeAk+d8)?WlruU zIAw|9Qxb;$x9Zhr=uug(PnrUio&rhP3GsfDU&dwM*AkpFZ7Cq#(XH5T{9r&l#}uCS zt!C4Iu`_A$ko2EbDAf7PTH%532WNC3L&qM&r%lEQ>nMA#A zkLt{D{nW$~B3?cF4>ZsYJps)kv&$ZPXqKzbRlB05SO4~E9-YD*-d$LmJ zGG!RmUH~&P>h%p!Ht8o&M|8Gwc zGLVFaADNm5R~qmelB9&#<{YgYfJThp2ON9UaD6;1dHbtM|h}w8L30Y;OdVq?9nt8Ie1f@XHHakf8ySepx(3t7{ zDd>#)`C6Udy}2T8-9OkGZXzVP4kj3!>|uJHf@<7;t$W1s@}hFdLjma^ePj3CGiCRW zUGx9?y`pV3)8iI{Mv2@kYzsFJt?DN!wF7sbpGr}x{yKs`fp9dKzqV7G7&-dvF^*GN zS*J((bAFVBAxVw2($AWQLsavbiWpQdr6LQIgToP*UvhPR@Pj1DfgXNt?HdRZ8@Ww+ z)UqFxYg!;%y?(6{1k!b+3y<^yM~htJ2%ON3ny#(j3+rEA=W|Ttp_HBBfVm$Pbkuj@ zz2#VMpPGpAxVXNlJS_TYbQxu%W;W}aP6M`4--h1IIhBRg&f;7Fe5mqoUt{i)`6}%a z%ehkbrg)Z1rv@02Au<}8noy%Nwi;7=s+I*%)PW^n-??#zayT~;B}0X#$^2G2P36Hx zsR1|ei%BlC;}pu9_i*y*+H||IiALX|VwEB7K+rhOd9($~UYmzV^%6gJh}oD1Q@Sr= z7Z+VGEzjC`;v9r(%X{_#qhVDrTpW(P?n63e{J9{KT^E#n@|?fYh0f9L1nFsV0WKhq zY+x+6es}-*EnlYA+sz(U6DdCE3Ldq@>GA~|D$h2tBl8wxq2!Z3*13-sfcQA#Vo6O; zy7CgW(0j)rDpW3pTDBeX|Jl6X!p0&Qv%+i z)e-}RefLKB(D#(SRq3Q(xzB9nby!P8aYv?2$rTSJMOnhsJI-#%LHds@0(tcTxV37K z$=6zT>HKzTNj_)i8}%T@XWGe?z(sXV1?V*)JF06JfQzXHNFq~bw~N?2;L10y$C^hx z@GAs8{hDp%JNHO^eZGM0wb|xGXpd|u&8r0`I@wRE`D!g~UXJv!3wDDlI#VN;z5Q8hgzi4U$3|ZrX?kc4C<%Kqf`>*y)GFw+Uo))1J!PjOW|0u!D zGHkkDwKPFKc|imQCtnx+O@fL9aRlp+XBq0y? z)?v${0(uvRl4aYCm7?P0OjbZj;i$M|2|)1yF#e7iWTq89=k5tUElCEum67=u!2889 zw46is`s5RnZ>NhexuTnpWBW5$E-Iu+1!6;Vd};ewFHtJA$6!jQatbx`lGdpA=1_!a zf_1T9t;JW-SNddkuU*bMXbnHL-+>M^`q-_qSCsbK9=%M@!FXgW{qE3cZdIX{eK;t1 z@W$%uc;3KFqv7NTvh`_G^$wj~JFD@?N4}=85GVK;`&!31vojFER_&F3pz$G>Cu{>3 zFSNLYhN$wnAijA^Q|`wlCCR)O_CD<|NmavM2SR}26brZh;A{hIm52Jc2ya3}{T>B+mHd3YA@#zET0h#LP3A!8^lX&qq; z;G|KIe9oHnGJw(Y&y^juJli?|7;xfx9}ybD<&26zuWr>6HPGIIHQSr*f(I$1HNfoX zVauD11JPgliAlMJA+my~l3#0UM0H)B=z-(tkORi)B&t&tW#0~(zu#SF>xxP zC?2P3;k6&s&dEZ!gmdjY=A(d6QIwkhn}AR5U@}O7bCbC7=J+n# zibu8;93W#2Ro(w&%qNrP-L06-3Z3&BIuEUjzZ*owlzIKm?#?k0D^P;102dx zsJDzJ^A_8y+A8ZFDHbRfpoH0bjXxMpzBTqPQG=}WRD2uv5KU z{n6cmD@lpopLHZ+65NVjSOF-~7vttGn66LK6G;;B0cGO2bLu6|?)=^YQo-Nvu0LG} zZ2SxWsYM9I*HW*PWz;|!4(}Kp2t|}$5lMIH@DaULP&R3lFI20(`k-pN zG-V_sTLu_Z(Mn1j{qhVLAVMhTwD+7OiRIivlB9E94LR!coHQM%o``Xof3@4JV9OpG zbB`I}A3FubPDaG_!gcNM%@~Ho_NyoQbvO-txjHEgO76+*RB%hojk#P`y?Dqpb9hM& z+aFa(Y^!?ytaa`5+4XKtQ|$%POetkg;=*|#G;erMFGZUe8~~Z&llCE}bsxsZ3%*sQ ziEe#9U1IKeyzfwa#L>4!RFCe;DO4RcA4CE^+2Ol=Aku@_OEZYa_w}Zh;pgy+jS}~?$07`HnH!^3!iiiIWL4Ue zEuX~#Bl^8lKGf|8bZ3E#bEFxjlD3))9>!^qWJhGe<3A<_h7h;r8QR0Rv+}M=I0~-) zewI6ZA+v8Ve?EL=JbN)7 zohMeEe%dNhDYVzvAtvkr!fOVYL`8&f@UKw;g+eLUtqhU)Cip6LX8KZR{;Ol|gJE)Y zaappr_lWUuyU@7mr&ZP6x@b@P2Msnz2@bm#?jT^-7Wi>LMreOO3@v}E%3)<2A5-T?`a&zW9Db^^S43KZ9ErWzRU=TwK*#R%gmu753o0 zgN@BomTNy+_HMh`X;x>>`GboC3;?%g!D$z_n$|ELKju4Z^Vx7|&+^L`noOmGspG8A znIVVQ*13i?Ob7{rb3*unEek_C*7okN-uk!iQqqH<z+gUi~$s+x(Fg18Q!Tv}P|fLdiDCseHdA0$3pEjIu8!SFc`M zErgQwzt_=_brT7`_ and `deciles >>> list(map(round, sat.quantiles(n=10))) [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +Monte Carlo inputs for simulations +********************************** + To estimate the distribution for a model than isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -963,6 +971,9 @@ Carlo simulation `_: >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP [1.4591308524824727, 1.8035946855390597, 2.175091447274739] +Approximating binomial distributions +************************************ + Normal distributions can be used to approximate `Binomial distributions `_ when the sample size is large and when the probability of a successful @@ -1000,6 +1011,10 @@ probability that the Python room will stay within its capacity limits? >>> mean(trial() <= k for i in range(10_000)) 0.8398 + +Naive bayesian classifier +************************* + Normal distributions commonly arise in machine learning problems. Wikipedia has a `nice example of a Naive Bayesian Classifier @@ -1054,6 +1069,48 @@ The final prediction goes to the largest posterior. This is known as the 'female' +Kernel density estimation +************************* + +It is possible to estimate a continuous probability density function +from a fixed number of discrete samples. + +The basic idea is to smooth the data using `a kernel function such as a +normal distribution, triangular distribution, or uniform distribution +`_. +The degree of smoothing is controlled by a single +parameter, ``h``, representing the variance of the kernel function. + +.. testcode:: + + import math + + def kde_normal(sample, h): + "Create a continous probability density function from a sample." + # Smooth the sample with a normal distribution of variance h. + kernel_h = NormalDist(0.0, math.sqrt(h)).pdf + n = len(sample) + def pdf(x): + return sum(kernel_h(x - x_i) for x_i in sample) / n + return pdf + +`Wikipedia has an example +`_ +where we can use the ``kde_normal()`` recipe to generate and plot +a probability density function estimated from a small sample: + +.. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde_normal(sample, h=2.25) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + +The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + +.. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; From cc2cf85d03cf29994a707aae5cc9a349a4165b84 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Sun, 13 Aug 2023 14:19:41 +0100 Subject: [PATCH 124/598] gh-107877: Update logging levels reference table with usage criteria. (#107894) Co-authored-by: Alex Waygood --- Doc/library/logging.rst | 48 ++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4c6e74ff66a11a..b582c918df0239 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -397,21 +397,39 @@ have specific values relative to the predefined levels. If you define a level with the same numeric value, it overwrites the predefined value; the predefined name is lost. -+-----------------------+---------------+ -| Level | Numeric value | -+=======================+===============+ -| .. py:data:: CRITICAL | 50 | -+-----------------------+---------------+ -| .. py:data:: ERROR | 40 | -+-----------------------+---------------+ -| .. py:data:: WARNING | 30 | -+-----------------------+---------------+ -| .. py:data:: INFO | 20 | -+-----------------------+---------------+ -| .. py:data:: DEBUG | 10 | -+-----------------------+---------------+ -| .. py:data:: NOTSET | 0 | -+-----------------------+---------------+ ++-----------------------+---------------+-------------------------------------+ +| Level | Numeric value | What it means / When to use it | ++=======================+===============+=====================================+ +| .. py:data:: NOTSET | 0 | When set on a logger, indicates that| +| | | ancestor loggers are to be consulted| +| | | to determine the effective level. | +| | | If that still resolves to | +| | | :const:`!NOTSET`, then all events | +| | | are logged. When set on a handler, | +| | | all events are handled. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: DEBUG | 10 | Detailed information, typically only| +| | | of interest to a developer trying to| +| | | diagnose a problem. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: INFO | 20 | Confirmation that things are working| +| | | as expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: WARNING | 30 | An indication that something | +| | | unexpected happened, or that a | +| | | problem might occur in the near | +| | | future (e.g. 'disk space low'). The | +| | | software is still working as | +| | | expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: ERROR | 40 | Due to a more serious problem, the | +| | | software has not been able to | +| | | perform some function. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: CRITICAL | 50 | A serious error, indicating that the| +| | | program itself may be unable to | +| | | continue running. | ++-----------------------+---------------+-------------------------------------+ .. _handler: From c3887b57a75a105615dd555aaf74e6c9a243ebdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joon=20Hwan=20=EA=B9=80=EC=A4=80=ED=99=98?= Date: Mon, 14 Aug 2023 18:26:18 +0900 Subject: [PATCH 125/598] gh-107910: Remove not needing newline in error message (GH-107928) --- Lib/test/test_descr.py | 4 ++-- Objects/typeobject.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index ad3eefba365856..fc8d0a305d8da0 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -989,8 +989,8 @@ class EditableScrollablePane(ScrollablePane,EditablePane): pass def test_mro_disagreement(self): # Testing error messages for MRO disagreement... - mro_err_msg = """Cannot create a consistent method resolution -order (MRO) for bases """ + mro_err_msg = ("Cannot create a consistent method resolution " + "order (MRO) for bases ") def raises(exc, expected, callable, *args): try: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 030e8bfc99b6d7..7ce3de4d58d0d3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2444,7 +2444,7 @@ set_mro_error(PyObject **to_merge, Py_ssize_t to_merge_size, int *remain) n = PyDict_GET_SIZE(set); off = PyOS_snprintf(buf, sizeof(buf), "Cannot create a \ -consistent method resolution\norder (MRO) for bases"); +consistent method resolution order (MRO) for bases"); i = 0; while (PyDict_Next(set, &i, &k, &v) && (size_t)off < sizeof(buf)) { PyObject *name = class_name(k); From 608927b01447b110de5094271fbc4d49c60130b0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:51:50 +0100 Subject: [PATCH 126/598] gh-103082: use IS_VALID_OPCODE instead of _PyOpcode_OpName to check if an opcode is defined (#107882) --- Python/instrumentation.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6d11649c07fbe4..b50e8e26476cd4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -7,6 +7,7 @@ #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode.h" +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -437,11 +438,10 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) static bool valid_opcode(int opcode) { - if (opcode > 0 && + if (IS_VALID_OPCODE(opcode) && + opcode != CACHE && opcode != RESERVED && - opcode < 255 && - _PyOpcode_OpName[opcode] && - _PyOpcode_OpName[opcode][0] != '<') + opcode < 255) { return true; } From 6fbaba552a52f93ecbe8be000888afa0b65b967e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 14 Aug 2023 16:37:44 +0200 Subject: [PATCH 127/598] gh-107938: Synchonise the signature of of sqlite3.connect and sqlite3.Connection.__init__ (#107939) --- Modules/_sqlite/clinic/_sqlite3.connect.c.h | 25 ++++++++++++++++++ Modules/_sqlite/connection.c | 29 ++++++++++++++++++++- Modules/_sqlite/module.c | 25 ++++++------------ 3 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 Modules/_sqlite/clinic/_sqlite3.connect.c.h diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h new file mode 100644 index 00000000000000..0c4ebdf0590c9e --- /dev/null +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -0,0 +1,25 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(pysqlite_connect__doc__, +"connect($module, /, database, timeout=5.0, detect_types=0,\n" +" isolation_level=\'\', check_same_thread=True,\n" +" factory=ConnectionType, cached_statements=128, uri=False, *,\n" +" autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL)\n" +"--\n" +"\n" +"Open a connection to the SQLite database file \'database\'.\n" +"\n" +"You can use \":memory:\" to open a database connection to a database that\n" +"resides in RAM instead of on disk."); + +#define PYSQLITE_CONNECT_METHODDEF \ + {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, +/*[clinic end generated code: output=6a8458c9edf8fb7f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index ddd7ace81198bb..3d2fcd3ae0788e 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -212,7 +212,6 @@ class sqlite3_int64_converter(CConverter): [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=dff8760fb1eba6a1]*/ -// NB: This needs to be in sync with the sqlite3.connect docstring /*[clinic input] _sqlite3.Connection.__init__ as pysqlite_connection_init @@ -346,6 +345,34 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, return -1; } +/*[clinic input] +# Create a new destination 'connect' for the docstring and methoddef only. +# This makes it possible to keep the signatures for Connection.__init__ and +# sqlite3.connect() synchronised. +output push +destination connect new file '{dirname}/clinic/_sqlite3.connect.c.h' + +# Only output the docstring and the PyMethodDef entry. +output everything suppress +output docstring_definition connect +output methoddef_define connect + +# Define the sqlite3.connect function by cloning Connection.__init__. +_sqlite3.connect as pysqlite_connect = _sqlite3.Connection.__init__ + +Open a connection to the SQLite database file 'database'. + +You can use ":memory:" to open a database connection to a database that +resides in RAM instead of on disk. +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=92260edff95d1720]*/ + +/*[clinic input] +# Restore normal Argument Clinic operation for the rest of this file. +output pop +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b899ba9273edcce7]*/ + #define VISIT_CALLBACK_CONTEXT(ctx) \ do { \ if (ctx) { \ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 368e581b4f3355..0c503dfbebbd6c 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -48,25 +48,16 @@ module _sqlite3 [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=81e330492d57488e]*/ -// NB: This needs to be in sync with the Connection.__init__ docstring. -PyDoc_STRVAR(module_connect_doc, -"connect($module, /, database, timeout=5.0, detect_types=0,\n" -" isolation_level='', check_same_thread=True,\n" -" factory=ConnectionType, cached_statements=128, uri=False, *,\n" -" autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL)\n" -"--\n" -"\n" -"Opens a connection to the SQLite database file database.\n" -"\n" -"You can use \":memory:\" to open a database connection to a database that resides\n" -"in RAM instead of on disk."); - -#define PYSQLITE_CONNECT_METHODDEF \ - {"connect", _PyCFunction_CAST(module_connect), METH_FASTCALL|METH_KEYWORDS, module_connect_doc}, +/* + * We create 'clinic/_sqlite3.connect.c.h' in connection.c, in order to + * keep the signatures of sqlite3.Connection.__init__ and + * sqlite3.connect() synchronised. + */ +#include "clinic/_sqlite3.connect.c.h" static PyObject * -module_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, - PyObject *kwnames) +pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, + PyObject *kwnames) { pysqlite_state *state = pysqlite_get_state(module); PyObject *factory = (PyObject *)state->ConnectionType; From 39745347f645ac99021f4ab981ff02ab5647b19c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 14 Aug 2023 19:36:29 +0100 Subject: [PATCH 128/598] gh-105481: reduce repetition in opcode metadata generation code (#107942) --- Include/internal/pycore_opcode_metadata.h | 29 +++--- Tools/cases_generator/generate_cases.py | 108 +++++++++------------- 2 files changed, 61 insertions(+), 76 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 9f4437c09e92cb..c0a2e9d91c4f49 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -56,11 +56,9 @@ #define _POP_JUMP_IF_TRUE 332 #define JUMP_TO_TOP 333 -#ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); -#else -int -_PyOpcode_num_popped(int opcode, int oparg, bool jump) { +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; @@ -500,13 +498,11 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return -1; } } -#endif +#endif // NEED_OPCODE_METADATA -#ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_pushed(int opcode, int oparg, bool jump); -#else -int -_PyOpcode_num_pushed(int opcode, int oparg, bool jump) { +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; @@ -946,7 +942,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return -1; } } -#endif +#endif // NEED_OPCODE_METADATA enum InstructionFormat { INSTR_FMT_IB, @@ -1004,11 +1000,8 @@ struct opcode_macro_expansion { #define OPCODE_UOP_NAME_SIZE 512 #define OPCODE_MACRO_EXPANSION_SIZE 256 -#ifndef NEED_OPCODE_METADATA extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; -extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; -extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; -#else // if NEED_OPCODE_METADATA +#ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1228,6 +1221,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [CACHE] = { true, INSTR_FMT_IX, 0 }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, }; +#endif // NEED_OPCODE_METADATA + +extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; +#ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, @@ -1357,6 +1354,10 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, }; +#endif // NEED_OPCODE_METADATA + +extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; +#ifdef NEED_OPCODE_METADATA const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [EXIT_TRACE] = "EXIT_TRACE", [SAVE_IP] = "SAVE_IP", diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index d35a16a80e8d00..e78b0edff9a0c8 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -4,6 +4,7 @@ """ import argparse +import contextlib import os import posixpath import sys @@ -141,6 +142,15 @@ def effect_str(effects: list[StackEffect]) -> str: typing.assert_never(thing) return instr, popped, pushed + @contextlib.contextmanager + def metadata_item(self, signature, open, close): + self.out.emit("") + self.out.emit(f"extern {signature};") + self.out.emit("#ifdef NEED_OPCODE_METADATA") + with self.out.block(f"{signature} {open}", close): + yield + self.out.emit("#endif // NEED_OPCODE_METADATA") + def write_stack_effect_functions(self) -> None: popped_data: list[tuple[AnyInstruction, str]] = [] pushed_data: list[tuple[AnyInstruction, str]] = [] @@ -156,25 +166,16 @@ def write_stack_effect_functions(self) -> None: def write_function( direction: str, data: list[tuple[AnyInstruction, str]] ) -> None: - self.out.emit("") - self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit( - f"extern int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump);" - ) - self.out.emit("#else") - self.out.emit("int") - self.out.emit( - f"_PyOpcode_num_{direction}(int opcode, int oparg, bool jump) {{" - ) - self.out.emit(" switch(opcode) {") - for instr, effect in data: - self.out.emit(f" case {instr.name}:") - self.out.emit(f" return {effect};") - self.out.emit(" default:") - self.out.emit(" return -1;") - self.out.emit(" }") - self.out.emit("}") - self.out.emit("#endif") + + with self.metadata_item( + f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", "", "" + ): + with self.out.block("switch(opcode)"): + for instr, effect in data: + self.out.emit(f"case {instr.name}:") + self.out.emit(f" return {effect};") + self.out.emit("default:") + self.out.emit(" return -1;") write_function("popped", popped_data) write_function("pushed", pushed_data) @@ -290,48 +291,33 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.out.emit("#define OPCODE_METADATA_SIZE 512") self.out.emit("#define OPCODE_UOP_NAME_SIZE 512") self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256") - self.out.emit("") - self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit( - "extern const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];" - ) - self.out.emit( - "extern const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];" - ) - self.out.emit( - "extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];" - ) - self.out.emit("#else // if NEED_OPCODE_METADATA") - self.out.emit( + with self.metadata_item( "const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {" - ) - - # Write metadata for each instruction - for thing in self.everything: - match thing: - case OverriddenInstructionPlaceHolder(): - continue - case parsing.InstDef(): - if thing.kind != "op": - self.write_metadata_for_inst(self.instrs[thing.name]) - case parsing.Macro(): - self.write_metadata_for_macro(self.macro_instrs[thing.name]) - case parsing.Pseudo(): - self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) - case _: - typing.assert_never(thing) - - # Write end of array - self.out.emit("};") + "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", + "=", + ";" + ): + # Write metadata for each instruction + for thing in self.everything: + match thing: + case OverriddenInstructionPlaceHolder(): + continue + case parsing.InstDef(): + if thing.kind != "op": + self.write_metadata_for_inst(self.instrs[thing.name]) + case parsing.Macro(): + self.write_metadata_for_macro(self.macro_instrs[thing.name]) + case parsing.Pseudo(): + self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) + case _: + typing.assert_never(thing) - with self.out.block( + with self.metadata_item( "const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] =", - ";", + "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", + "=", + ";" ): # Write macro expansion for each non-pseudo instruction for thing in self.everything: @@ -360,13 +346,11 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case _: typing.assert_never(thing) - with self.out.block( - "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";" + with self.metadata_item( + "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" ): self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') - self.out.emit("#endif // NEED_OPCODE_METADATA") - with open(pymetadata_filename, "w") as f: # Create formatter self.out = Formatter(f, 0, comment="#") @@ -511,7 +495,7 @@ def emit_metadata_entry(self, name: str, fmt: str, flags: InstructionFlags) -> N if not flag_names: flag_names.append("0") self.out.emit( - f" [{name}] = {{ true, {INSTR_FMT_PREFIX}{fmt}," + f"[{name}] = {{ true, {INSTR_FMT_PREFIX}{fmt}," f" {' | '.join(flag_names)} }}," ) From a2a4b9f1ec86b9762a5d35895ac5b528e03d5b98 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 14 Aug 2023 14:41:27 -0700 Subject: [PATCH 129/598] Attempt to speed up deepfreeze.py (#107887) * Instead of calling get_identifiers_and_strings(), extract identifiers and strings from pycore_global_strings.h. * Avoid ast.literal_eval(), it's very slow. --- Makefile.pre.in | 2 +- Tools/build/deepfreeze.py | 35 +++++++++++++++++++++++------------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 52236f7924503d..3a628bf49e97c1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1253,7 +1253,7 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) +DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) # BEGIN: deepfreeze modules Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index a11fe6a62811ab..ce609bd0898741 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -6,7 +6,6 @@ by Python 3.10, and 3.11 features are not available. """ import argparse -import ast import builtins import collections import contextlib @@ -17,10 +16,10 @@ from typing import Dict, FrozenSet, TextIO, Tuple import umarshal -from generate_global_objects import get_identifiers_and_strings + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) verbose = False -identifiers, strings = get_identifiers_and_strings() # This must be kept in sync with opcode.py RESUME = 151 @@ -114,6 +113,7 @@ def __init__(self, file: TextIO) -> None: self.hits, self.misses = 0, 0 self.finis: list[str] = [] self.inits: list[str] = [] + self.identifiers, self.strings = self.get_identifiers_and_strings() self.write('#include "Python.h"') self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') @@ -121,6 +121,19 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_long.h"') self.write("") + def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: + filename = os.path.join(ROOT, "Include", "internal", "pycore_global_strings.h") + with open(filename) as fp: + lines = fp.readlines() + identifiers: set[str] = set() + strings: dict[str, str] = {} + for line in lines: + if m := re.search(r"STRUCT_FOR_ID\((\w+)\)", line): + identifiers.add(m.group(1)) + if m := re.search(r'STRUCT_FOR_STR\((\w+), "(.*?)"\)', line): + strings[m.group(2)] = m.group(1) + return identifiers, strings + @contextlib.contextmanager def indent(self) -> None: save_level = self.level @@ -171,9 +184,9 @@ def generate_bytes(self, name: str, b: bytes) -> str: return f"& {name}.ob_base.ob_base" def generate_unicode(self, name: str, s: str) -> str: - if s in strings: - return f"&_Py_STR({strings[s]})" - if s in identifiers: + if s in self.strings: + return f"&_Py_STR({self.strings[s]})" + if s in self.identifiers: return f"&_Py_ID({s})" if len(s) == 1: c = ord(s) @@ -441,12 +454,10 @@ def is_frozen_header(source: str) -> bool: def decode_frozen_data(source: str) -> types.CodeType: - lines = source.splitlines() - while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None: - del lines[0] - while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: - del lines[-1] - values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip()) + values: list[int] = [] + for line in source.splitlines(): + if re.match(FROZEN_DATA_LINE, line): + values.extend([int(x) for x in line.split(",") if x.strip()]) data = bytes(values) return umarshal.loads(data) From 8d3cb1bc4b5de091d7b5fcc5ce7378151a8f4f45 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 14 Aug 2023 23:37:27 +0100 Subject: [PATCH 130/598] Convert the GitHub issue templates into GitHub issue forms (#107920) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Hugo van Kemenade Co-authored-by: Ezio Melotti --- .github/ISSUE_TEMPLATE/bug.md | 50 --------------------- .github/ISSUE_TEMPLATE/bug.yml | 69 +++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/crash.md | 48 -------------------- .github/ISSUE_TEMPLATE/crash.yml | 71 ++++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature.md | 50 --------------------- .github/ISSUE_TEMPLATE/feature.yml | 42 ++++++++++++++++++ 6 files changed, 182 insertions(+), 148 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug.md create mode 100644 .github/ISSUE_TEMPLATE/bug.yml delete mode 100644 .github/ISSUE_TEMPLATE/crash.md create mode 100644 .github/ISSUE_TEMPLATE/crash.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature.md create mode 100644 .github/ISSUE_TEMPLATE/feature.yml diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md deleted file mode 100644 index 47037cd319e7e2..00000000000000 --- a/.github/ISSUE_TEMPLATE/bug.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Bug report -about: Submit a bug report -labels: "type-bug" ---- - - - -# Bug report - -## Checklist - - - -- [ ] I am confident this is a bug in CPython, not a bug in a third-party project -- [ ] I have searched the CPython issue tracker, and am confident this bug has not been reported before - -## A clear and concise description of the bug - - - - - -# Your environment - - - -- CPython versions tested on: -- Operating system and architecture: - - diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000000000..05f4f317ccc97e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,69 @@ +name: Bug report +description: Submit a bug report +labels: ["type-bug"] +body: + - type: markdown + attributes: + value: | + **New to Python?** + + For help or advice on using Python, try one of the following options instead of opening a GitHub issue: + + - Posting on [Discourse](https://discuss.python.org/c/users/7) + - Reading the [Python tutorial](https://docs.python.org/3/tutorial/) + - Emailing [python-list](https://mail.python.org/mailman/listinfo/python-list) + - type: checkboxes + attributes: + label: Checklist + description: A bug in a third-party project (for example, `pip` or `requests`) should be reported to that project's issue tracker, not CPython + options: + - label: I am confident this is a bug in CPython, not a bug in a third-party project + required: false + - label: | + I have searched the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc), + and am confident this bug has not been reported before + required: false + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false + - type: input + attributes: + label: "Output from running 'python -VV' on the command line:" + description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. + validations: + required: false + - type: textarea + attributes: + label: "A clear and concise description of the bug:" + description: > + Tell us what happened. + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + Put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md deleted file mode 100644 index a268249d1c1e65..00000000000000 --- a/.github/ISSUE_TEMPLATE/crash.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: Crash report -about: A hard crash of the interpreter, possibly with a core dump -labels: "type-crash" ---- - - - -# Crash report - - - - - -# Error messages - - - - - -# Your environment - - - -- CPython versions tested on: -- Operating system and architecture: - - diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml new file mode 100644 index 00000000000000..1ea84b893697a6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -0,0 +1,71 @@ +name: Crash report +description: A hard crash of the interpreter, possibly with a core dump +labels: ["type-crash"] +body: + - type: markdown + attributes: + value: | + This form is for hard crashes of the Python interpreter, segmentation faults, failed C-level assertions, and similar. Unexpected exceptions raised from Python functions in the standard library count as bugs rather than crashes. + + The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false + - type: input + attributes: + label: "Output from running 'python -VV' on the command line:" + description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. + validations: + required: false + - type: textarea + attributes: + label: What happened? + description: > + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + Put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: textarea + attributes: + label: Error messages + description: > + Enter any error messages caused by the crash, including a core dump if there is one. + Feel free to leave this bit blank if it isn't relevant. + placeholder: | + Error messages should be formatted like this: + +

+ Error messages/core dump + + ``` + # paste errors here, if you have any + ``` +
+ validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md deleted file mode 100644 index 7e96bc9df665c2..00000000000000 --- a/.github/ISSUE_TEMPLATE/feature.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Feature or enhancement -about: Submit a proposal for a new CPython feature or enhancement -labels: "type-feature" ---- - - - -# Feature or enhancement - - - - - -# Pitch - - - - - -# Previous discussion - - - - - - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000000000..a1c48bbff829c4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,42 @@ +name: Feature or enhancement +description: Submit a proposal for a new CPython feature or enhancement +labels: ["type-feature"] +body: + - type: markdown + attributes: + value: | + # Proposing a feature to CPython? + + You'll need to demonstrate widespread support for your idea among the community. + + Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. + - type: checkboxes + attributes: + label: Has this already been discussed elsewhere? + options: + - label: I have already discussed this feature proposal on Discourse + - label: This is a minor feature, which does not need previous discussion elsewhere + - type: textarea + attributes: + label: "Links to previous discussion of this feature:" + validations: + required: false + - type: input + attributes: + label: "Summary of proposal:" + description: A one-line summary of your proposal. + validations: + required: true + - type: textarea + attributes: + label: "Pitch:" + description: > + Explain why this feature or enhancement should be implemented and how it would be used. + Add examples, if applicable. + Put any code blocks inside triple backticks. + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true From 580f357c663bd704a0903e4174cdf458cac56416 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 15 Aug 2023 08:25:57 +0900 Subject: [PATCH 131/598] =?UTF-8?q?Revert=20"gh-104469=20:=20Convert=20=5F?= =?UTF-8?q?testcapi/vectorcall=5Flimited.c=20to=20use=20AC=20=E2=80=A6=20(?= =?UTF-8?q?gh-107951)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "gh-104469 : Convert _testcapi/vectorcall_limited.c to use AC (gh-107857)" This reverts commit 2e27da18952c6561f48dab706b5911135cedd7cf. --- .../_testcapi/clinic/vectorcall_limited.c.h | 42 ------------------- Modules/_testcapi/vectorcall_limited.c | 30 ++----------- 2 files changed, 4 insertions(+), 68 deletions(-) delete mode 100644 Modules/_testcapi/clinic/vectorcall_limited.c.h diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h deleted file mode 100644 index bc89f58b65739b..00000000000000 --- a/Modules/_testcapi/clinic/vectorcall_limited.c.h +++ /dev/null @@ -1,42 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -#if defined(LIMITED_API_AVAILABLE) - -PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, -"call_vectorcall($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ - {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, - -#endif /* defined(LIMITED_API_AVAILABLE) */ - -#if defined(LIMITED_API_AVAILABLE) - -PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, -"call_vectorcall_method($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ - {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, - -#endif /* defined(LIMITED_API_AVAILABLE) */ - -#ifndef _TESTCAPI_CALL_VECTORCALL_METHODDEF - #define _TESTCAPI_CALL_VECTORCALL_METHODDEF -#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHODDEF) */ - -#ifndef _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF - #define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF -#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF) */ -/*[clinic end generated code: output=409028b637aba77b input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index 9f02c7afa7c971..a96925e840121a 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,6 +1,5 @@ #define Py_LIMITED_API 0x030c0000 // 3.12 #include "parts.h" -#include "clinic/vectorcall_limited.c.h" #ifdef LIMITED_API_AVAILABLE @@ -8,11 +7,6 @@ /* Test Vectorcall in the limited API */ -/*[clinic input] -module _testcapi -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ - static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { return PyUnicode_FromString("tp_call called"); @@ -38,16 +32,8 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) return self; } -/*[clinic input] -_testcapi.call_vectorcall - - callable: object - / -[clinic start generated code]*/ - static PyObject * -_testcapi_call_vectorcall(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ +call_vectorcall(PyObject* self, PyObject *callable) { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -91,16 +77,8 @@ _testcapi_call_vectorcall(PyObject *module, PyObject *callable) return result; } -/*[clinic input] -_testcapi.call_vectorcall_method - - callable: object - / -[clinic start generated code]*/ - static PyObject * -_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ +call_vectorcall_method(PyObject* self, PyObject *callable) { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -175,8 +153,8 @@ static PyType_Spec LimitedVectorCallClass_spec = { }; static PyMethodDef TestMethods[] = { - _TESTCAPI_CALL_VECTORCALL_METHODDEF - _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + {"call_vectorcall", call_vectorcall, METH_O}, + {"call_vectorcall_method", call_vectorcall_method, METH_O}, {NULL}, }; From a482e5bf0022f85424a6308529a9ad51f1bfbb71 Mon Sep 17 00:00:00 2001 From: Romuald Brunet Date: Tue, 15 Aug 2023 09:23:54 +0200 Subject: [PATCH 132/598] gh-76913: Add "merge extras" feature to LoggerAdapter (GH-107292) --- Doc/library/logging.rst | 11 ++++- Lib/logging/__init__.py | 18 ++++++++- Lib/test/test_logging.py | 40 +++++++++++++++++++ ...3-08-14-17-15-59.gh-issue-76913.LLD0rT.rst | 1 + 4 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index b582c918df0239..49e870e9e2479b 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1002,10 +1002,14 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra) +.. class:: LoggerAdapter(logger, extra, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. + underlying :class:`Logger` instance, a dict-like object (*extra*), and a + boolean (*merge_extra*) indicating whether or not the *extra* argument of + individual log calls should be merged with the :class:`LoggerAdapter` extra. + The default behavior is to ignore the *extra* argument of individual log + calls and only use the one of the :class:`LoggerAdapter` instance .. method:: process(msg, kwargs) @@ -1037,6 +1041,9 @@ interchangeably. Remove the undocumented ``warn()`` method which was an alias to the ``warning()`` method. +.. versionchanged:: 3.13 + The *merge_extra* argument was added. + Thread Safety ------------- diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 527fc5c631730a..2d228e563094c8 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1879,7 +1879,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra=None): + def __init__(self, logger, extra=None, merge_extra=False): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1889,9 +1889,20 @@ def __init__(self, logger, extra=None): following example: adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) + + By default, LoggerAdapter objects will drop the "extra" argument + passed on the individual log calls to use its own instead. + + Initializing it with merge_extra=True will instead merge both + maps when logging, the individual call extra taking precedence + over the LoggerAdapter instance extra + + .. versionchanged:: 3.13 + The *merge_extra* argument was added. """ self.logger = logger self.extra = extra + self.merge_extra = merge_extra def process(self, msg, kwargs): """ @@ -1903,7 +1914,10 @@ def process(self, msg, kwargs): Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ - kwargs["extra"] = self.extra + if self.merge_extra and "extra" in kwargs: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} + else: + kwargs["extra"] = self.extra return msg, kwargs # diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index def976fbe96ba3..f26846f9663e5c 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5433,6 +5433,46 @@ def process(self, msg, kwargs): self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) + def test_extra_in_records(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}) + + self.adapter.critical('foo should be here') + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '1') + + def test_extra_not_merged_by_default(self): + self.adapter.critical('foo should NOT be here', extra={'foo': 'nope'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertFalse(hasattr(record, 'foo')) + + def test_extra_merged(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo and bar should be here', extra={'bar': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertTrue(hasattr(record, 'bar')) + self.assertEqual(record.foo, '1') + self.assertEqual(record.bar, '2') + + def test_extra_merged_log_call_has_precedence(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo shall be min', extra={'foo': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '2') + class LoggerTest(BaseTest, AssertErrorMessage): diff --git a/Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst b/Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst new file mode 100644 index 00000000000000..5f9a84e714ae20 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst @@ -0,0 +1 @@ +Add *merge_extra* parameter/feature to :class:`logging.LoggerAdapter` From 13c36dc9ae5240124932137de4a94d81292c6c5f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 15 Aug 2023 10:09:56 +0200 Subject: [PATCH 133/598] gh-93057: Deprecate positional use of optional sqlite3.connect() params (#107948) --- Doc/library/sqlite3.rst | 6 ++++ Doc/whatsnew/3.13.rst | 5 +++ Lib/test/test_sqlite3/test_dbapi.py | 13 +++++++ Lib/test/test_sqlite3/test_factory.py | 11 +++++- ...3-08-14-19-49-02.gh-issue-93057.5nJwO5.rst | 3 ++ Modules/_sqlite/clinic/_sqlite3.connect.c.h | 10 ++++-- Modules/_sqlite/clinic/connection.c.h | 35 ++++++++++++++++++- Modules/_sqlite/connection.c | 3 +- Modules/_sqlite/module.c | 11 ++++++ 9 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 50344058c26041..5f1676e5ae2f56 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -355,6 +355,12 @@ Module functions .. versionadded:: 3.12 The *autocommit* parameter. + .. versionchanged:: 3.13 + Positional use of the parameters *timeout*, *detect_types*, + *isolation_level*, *check_same_thread*, *factory*, *cached_statements*, + and *uri* is deprecated. + They will become keyword-only parameters in Python 3.15. + .. function:: complete_statement(statement) Return ``True`` if the string *statement* appears to contain diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 63cdee6cf1a4f3..a65a98bae13579 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -219,6 +219,11 @@ Deprecated They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) +* Passing more than one positional argument to :func:`sqlite3.connect` and the + :class:`sqlite3.Connection` constructor is deprecated. The remaining + parameters will become keyword-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`107948`.) + Pending Removal in Python 3.14 ------------------------------ diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 3f9bd0248a8b96..c9a9e1353938c6 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -582,6 +582,19 @@ def test_connection_config(self): with self.assertRaisesRegex(sqlite.IntegrityError, "constraint"): cx.execute("insert into u values(0)") + def test_connect_positional_arguments(self): + regex = ( + r"Passing more than 1 positional argument to sqlite3.connect\(\)" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + sqlite.connect(":memory:", 1.0) + self.assertEqual(cm.filename, __file__) + + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 7c36135ecadccd..d63589483e1042 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -66,7 +66,16 @@ class Factory(sqlite.Connection): def __init__(self, *args, **kwargs): super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) + regex = ( + r"Passing more than 1 positional argument to _sqlite3.Connection\(\) " + r"is deprecated. Parameters 'timeout', 'detect_types', " + r"'isolation_level', 'check_same_thread', 'factory', " + r"'cached_statements' and 'uri' will become keyword-only " + r"parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) + self.assertEqual(cm.filename, __file__) self.assertIsNone(con.isolation_level) self.assertIsInstance(con, Factory) diff --git a/Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst b/Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst new file mode 100644 index 00000000000000..6a4feaac25bc11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst @@ -0,0 +1,3 @@ +Passing more than one positional argument to :func:`sqlite3.connect` and the +:class:`sqlite3.Connection` constructor is deprecated. The remaining parameters +will become keyword-only in Python 3.15. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index 0c4ebdf0590c9e..998c8de1e09f16 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -18,8 +18,14 @@ PyDoc_STRVAR(pysqlite_connect__doc__, "Open a connection to the SQLite database file \'database\'.\n" "\n" "You can use \":memory:\" to open a database connection to a database that\n" -"resides in RAM instead of on disk."); +"resides in RAM instead of on disk.\n" +"\n" +"Note: Passing more than 1 positional argument to _sqlite3.connect() is\n" +"deprecated. Parameters \'timeout\', \'detect_types\', \'isolation_level\',\n" +"\'check_same_thread\', \'factory\', \'cached_statements\' and \'uri\' will\n" +"become keyword-only parameters in Python 3.15.\n" +""); #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=6a8458c9edf8fb7f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d49736db880f09a input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index e869d7d9e9384c..af98d61ea7cccc 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -59,6 +59,39 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) int uri = 0; enum autocommit_mode autocommit = LEGACY_TRANSACTION_CONTROL; + // Emit compiler warnings when we get to Python 3.15. + #if PY_VERSION_HEX >= 0x030f00C0 + # error \ + "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ + "'isolation_level', 'check_same_thread', 'factory', " \ + "'cached_statements' and 'uri' in the clinic input of " \ + "'_sqlite3.Connection.__init__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030f00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ + "'isolation_level', 'check_same_thread', 'factory', " \ + "'cached_statements' and 'uri' in the clinic input of " \ + "'_sqlite3.Connection.__init__' to be keyword-only.") + # else + # warning \ + "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ + "'isolation_level', 'check_same_thread', 'factory', " \ + "'cached_statements' and 'uri' in the clinic input of " \ + "'_sqlite3.Connection.__init__' to be keyword-only." + # endif + #endif + if (nargs > 1 && nargs <= 8) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to _sqlite3.Connection()" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15.", 1)) + { + goto exit; + } + } fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 8, 0, argsbuf); if (!fastargs) { goto exit; @@ -1659,4 +1692,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=d3c6cb9326736ea5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5a05e5294ad9d2ce input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3d2fcd3ae0788e..0819acd0940964 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -216,6 +216,7 @@ class sqlite3_int64_converter(CConverter): _sqlite3.Connection.__init__ as pysqlite_connection_init database: object + * [from 3.15] timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" @@ -234,7 +235,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int check_same_thread, PyObject *factory, int cache_size, int uri, enum autocommit_mode autocommit) -/*[clinic end generated code: output=cba057313ea7712f input=9b0ab6c12f674fa3]*/ +/*[clinic end generated code: output=cba057313ea7712f input=219c3dbecbae7d99]*/ { if (PySys_Audit("sqlite3.connect", "O", database) < 0) { return -1; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 0c503dfbebbd6c..dd45ffc1988f7e 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -64,6 +64,17 @@ pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, static const int FACTORY_POS = 5; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (nargs > 1 && nargs <= 8) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to sqlite3.connect()" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15.", 1)) + { + return NULL; + } + } if (nargs > FACTORY_POS) { factory = args[FACTORY_POS]; } From d66bc9e8a7a8d6774d912a4b9d151885c4d8de1d Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 15 Aug 2023 14:26:42 +0300 Subject: [PATCH 134/598] gh-107967: Fix infinite recursion on invalid escape sequence warning (#107968) --- Lib/test/test_fstring.py | 10 ++++++++++ Parser/tokenizer.c | 3 +++ 2 files changed, 13 insertions(+) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index cb14bba2602def..16f01973f99f3e 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1673,5 +1673,15 @@ def test_debug_in_file(self): self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'), "3\n=3") + def test_syntax_warning_infinite_recursion_in_file(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'w') as f: + f.write(r"print(f'\{1}')") + + _, stdout, stderr = assert_python_ok(script) + self.assertIn(rb'\1', stdout) + self.assertEqual(len(stderr.strip().splitlines()), 2) + if __name__ == '__main__': unittest.main() diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5a42f6f357317f..b10c9f1f8ea2cc 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1539,6 +1539,9 @@ parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) static int warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { + if (!tok->report_warnings) { + return 0; + } PyObject *msg = PyUnicode_FromFormat( "invalid escape sequence '\\%c'", From 6515ec3d3d5acd3d0b99c88794bdec09f0831e5b Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 15 Aug 2023 22:58:12 +0900 Subject: [PATCH 135/598] gh-107963: Fix set_forkserver_preload to check the type of given list (#107965) gh-107963: Fix set_forkserver_preload to check the type of given list --- Lib/multiprocessing/forkserver.py | 2 +- Lib/test/_test_multiprocessing.py | 8 ++++++++ .../2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 22a911a7a29cdc..4642707dae2f4e 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -61,7 +61,7 @@ def _stop_unlocked(self): def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): + if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f881a5d4674699..10754964e73bc5 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5369,6 +5369,14 @@ def test_context(self): self.assertRaises(ValueError, ctx.set_start_method, None) self.check_context(ctx) + def test_context_check_module_types(self): + try: + ctx = multiprocessing.get_context('forkserver') + except ValueError: + raise unittest.SkipTest('forkserver should be available') + with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): + ctx.set_forkserver_preload([1, 2, 3]) + def test_set_get(self): multiprocessing.set_forkserver_preload(PRELOAD) count = 0 diff --git a/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst new file mode 100644 index 00000000000000..3a73b2da0c4334 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst @@ -0,0 +1,2 @@ +Fix :func:`multiprocessing.set_forkserver_preload` to check the given list +of modules names. Patch by Dong-hee Na. From e90036c9bdf25621c15207a9cabd119fb5366c27 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 15 Aug 2023 16:00:52 +0200 Subject: [PATCH 136/598] gh-107880: Argument Clinic: Fix regression in gh-107885 (#107974) Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 35 ++++++++++++++++++++++++++++++++--- Tools/clinic/clinic.py | 33 +++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f067a26d1fb3ae..a67cd301f6eaa8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -609,6 +609,35 @@ def test_directive_output_invalid_command(self): """ self.expect_failure(block, err, lineno=2) + def test_validate_cloned_init(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + @classmethod + C.__init__ = C.meth + [clinic start generated code]*/ + """ + err = "'__init__' must be a normal method, not a class or static method" + self.expect_failure(block, err, lineno=8) + + def test_validate_cloned_new(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + C.__new__ = C.meth + [clinic start generated code]*/ + """ + err = "'__new__' must be a class method" + self.expect_failure(block, err, lineno=7) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( @@ -1918,7 +1947,7 @@ class Foo "" "" self.parse_function(block) def test_new_must_be_a_class_method(self): - err = "__new__ must be a class method!" + err = "'__new__' must be a class method!" block = """ module foo class Foo "" "" @@ -1927,7 +1956,7 @@ class Foo "" "" self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - err = "__init__ must be a normal method, not a class or static method!" + err = "'__init__' must be a normal method, not a class or static method!" block = """ module foo class Foo "" "" @@ -2030,7 +2059,7 @@ def test_illegal_c_identifier(self): self.expect_failure(block, err, lineno=2) def test_cannot_convert_special_method(self): - err = "__len__ is a special method and cannot be converted" + err = "'__len__' is a special method and cannot be converted" block = """ class T "" "" T.__len__ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1e0303c77087eb..11dbfb3fbe858e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4840,6 +4840,21 @@ def state_dsl_start(self, line: str) -> None: self.next(self.state_modulename_name, line) + def update_function_kind(self, fullname: str) -> None: + fields = fullname.split('.') + name = fields.pop() + _, cls = self.clinic._module_and_class(fields) + if name in unsupported_special_methods: + fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") + if name == '__new__': + if (self.kind is not CLASS_METHOD) or (not cls): + fail("'__new__' must be a class method!") + self.kind = METHOD_NEW + elif name == '__init__': + if (self.kind is not CALLABLE) or (not cls): + fail("'__init__' must be a normal method, not a class or static method!") + self.kind = METHOD_INIT + def state_modulename_name(self, line: str) -> None: # looking for declaration, which establishes the leftmost column # line should be @@ -4888,6 +4903,7 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) + self.update_function_kind(full_name) overrides: dict[str, Any] = { "name": function_name, "full_name": full_name, @@ -4948,20 +4964,9 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) - fields = full_name.split('.') - if fields[-1] in unsupported_special_methods: - fail(f"{fields[-1]} is a special method and cannot be converted to Argument Clinic! (Yet.)") - - if fields[-1] == '__new__': - if (self.kind is not CLASS_METHOD) or (not cls): - fail("__new__ must be a class method!") - self.kind = METHOD_NEW - elif fields[-1] == '__init__': - if (self.kind is not CALLABLE) or (not cls): - fail("__init__ must be a normal method, not a class or static method!") - self.kind = METHOD_INIT - if not return_converter: - return_converter = init_return_converter() + self.update_function_kind(full_name) + if self.kind is METHOD_INIT and not return_converter: + return_converter = init_return_converter() if not return_converter: return_converter = CReturnConverter() From 607f18c89456cdc9064e27f86a7505e011209757 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 15 Aug 2023 16:41:40 +0200 Subject: [PATCH 137/598] gh-107972: Argument Clinic: Ensure a C basename is provided after 'as' (#107973) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/test/test_clinic.py | 30 ++++++++++++++++++++++++++---- Tools/clinic/clinic.py | 9 ++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a67cd301f6eaa8..896730571d8152 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -28,7 +28,8 @@ def _make_clinic(*, filename='clinic_tests'): return c -def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, + strip=True): """Helper for the parser tests. tc: unittest.TestCase; passed self in the wrapper @@ -38,7 +39,9 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): filename: str, optional filename lineno: int, optional line number """ - code = dedent(code).strip() + code = dedent(code) + if strip: + code = code.strip() errmsg = re.escape(errmsg) with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: parser(code) @@ -638,6 +641,19 @@ class C "void *" "" err = "'__new__' must be a class method" self.expect_failure(block, err, lineno=7) + def test_no_c_basename_cloned(self): + block = """ + /*[clinic input] + foo2 + [clinic start generated code]*/ + /*[clinic input] + foo as = foo2 + [clinic start generated code]*/ + """ + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, lineno=5) + + class ParseFileUnitTest(TestCase): def expect_parsing_failure( @@ -857,9 +873,10 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): assert isinstance(s[function_index], clinic.Function) return s[function_index] - def expect_failure(self, block, err, *, filename=None, lineno=None): + def expect_failure(self, block, err, *, + filename=None, lineno=None, strip=True): return _expect_failure(self, self.parse_function, block, err, - filename=filename, lineno=lineno) + filename=filename, lineno=lineno, strip=strip) def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) @@ -1520,6 +1537,11 @@ def test_illegal_c_basename(self): err = "Illegal C basename: '935'" self.expect_failure(block, err) + def test_no_c_basename(self): + block = "foo as " + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, strip=False) + def test_single_star(self): block = """ module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 11dbfb3fbe858e..6ff2622d33b381 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4879,9 +4879,11 @@ def state_modulename_name(self, line: str) -> None: before, equals, existing = line.rpartition('=') c_basename: str | None if equals: - full_name, _, c_basename = before.partition(' as ') + full_name, as_, c_basename = before.partition(' as ') full_name = full_name.strip() c_basename = c_basename.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") existing = existing.strip() if (is_legal_py_identifier(full_name) and (not c_basename or is_legal_c_identifier(c_basename)) and @@ -4932,10 +4934,11 @@ def state_modulename_name(self, line: str) -> None: line, _, returns = line.partition('->') returns = returns.strip() - full_name, _, c_basename = line.partition(' as ') + full_name, as_, c_basename = line.partition(' as ') full_name = full_name.strip() c_basename = c_basename.strip() or None - + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") if not is_legal_py_identifier(full_name): fail(f"Illegal function name: {full_name!r}") if c_basename and not is_legal_c_identifier(c_basename): From 09322724319d4c23195300b222a1c0ea720af56b Mon Sep 17 00:00:00 2001 From: Finn Womack Date: Tue, 15 Aug 2023 08:33:00 -0700 Subject: [PATCH 138/598] gh-106242: Fix path truncation in os.path.normpath (GH-106816) --- Include/internal/pycore_fileutils.h | 1 + Lib/test/test_genericpath.py | 4 +++ ...-08-14-23-11-11.gh-issue-106242.71HMym.rst | 1 + Modules/posixmodule.c | 4 ++- Python/fileutils.c | 29 ++++++++++++++----- 5 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index daa32c0dff6097..cb5ac76772de5e 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -260,6 +260,7 @@ extern int _Py_add_relfile(wchar_t *dirname, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 489044f8090d3b..4f311c2d498e9f 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -460,6 +460,10 @@ def test_normpath_issue5827(self): for path in ('', '.', '/', '\\', '///foo/.//bar//'): self.assertIsInstance(self.pathmodule.normpath(path), str) + def test_normpath_issue106242(self): + for path in ('\x00', 'foo\x00bar', '\x00\x00', '\x00foo', 'foo\x00'): + self.assertEqual(self.pathmodule.normpath(path), path) + def test_abspath_issue3426(self): # Check that abspath returns unicode when the arg is unicode # with both ASCII and non-ASCII cwds. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst new file mode 100644 index 00000000000000..44237a9f15708c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst @@ -0,0 +1 @@ +Fixes :func:`os.path.normpath` to handle embedded null characters without truncating the path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a42e41c081e5b7..c391aab6fdce1d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5274,7 +5274,9 @@ os__path_normpath_impl(PyObject *module, PyObject *path) if (!buffer) { return NULL; } - PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); + PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); PyMem_Free(buffer); return result; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 19b23f6bd18b30..0ffd9885fdc846 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2377,12 +2377,14 @@ _Py_find_basename(const wchar_t *filename) path, which will be within the original buffer. Guaranteed to not make the path longer, and will not fail. 'size' is the length of the path, if known. If -1, the first null character will be assumed - to be the end of the path. */ + to be the end of the path. 'normsize' will be set to contain the + length of the resulting normalized path. */ wchar_t * -_Py_normpath(wchar_t *path, Py_ssize_t size) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) { assert(path != NULL); - if (!path[0] || size == 0) { + if (!path[0] && size < 0 || size == 0) { + *normsize = 0; return path; } wchar_t *pEnd = size >= 0 ? &path[size] : NULL; @@ -2431,11 +2433,7 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) *p2++ = lastC = *p1; } } - if (sepCount) { - minP2 = p2; // Invalid path - } else { - minP2 = p2 - 1; // Absolute path has SEP at minP2 - } + minP2 = p2 - 1; } #else // Skip past two leading SEPs @@ -2495,13 +2493,28 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) while (--p2 != minP2 && *p2 == SEP) { *p2 = L'\0'; } + } else { + --p2; } + *normsize = p2 - path + 1; #undef SEP_OR_END #undef IS_SEP #undef IS_END return path; } +/* In-place path normalisation. Returns the start of the normalized + path, which will be within the original buffer. Guaranteed to not + make the path longer, and will not fail. 'size' is the length of + the path, if known. If -1, the first null character will be assumed + to be the end of the path. */ +wchar_t * +_Py_normpath(wchar_t *path, Py_ssize_t size) +{ + Py_ssize_t norm_length; + return _Py_normpath_and_size(path, size, &norm_length); +} + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. From 971a4c27517e76155a4a489096eba69c53a7b9e9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 15 Aug 2023 16:40:05 +0100 Subject: [PATCH 139/598] gh-103082: remove assumption that INSTRUMENTED_LINE is the last instrumented opcode (#107978) --- Python/instrumentation.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b50e8e26476cd4..48befed4ea838c 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -121,7 +121,7 @@ static inline bool opcode_has_event(int opcode) { return ( - opcode < INSTRUMENTED_LINE && + opcode != INSTRUMENTED_LINE && INSTRUMENTED_OPCODES[opcode] > 0 ); } @@ -1202,7 +1202,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, Py_DECREF(line_obj); done: assert(original_opcode != 0); - assert(original_opcode < INSTRUMENTED_LINE); + assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } From 34e1917912f05e3ab5c9b1e39f678bd36388499e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 15 Aug 2023 18:02:32 +0100 Subject: [PATCH 140/598] gh-106242: Minor fixup to avoid compiler warnings (GH-107983) --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index 0ffd9885fdc846..9c4998397c4ac8 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2383,7 +2383,7 @@ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) { assert(path != NULL); - if (!path[0] && size < 0 || size == 0) { + if ((size < 0 && !path[0]) || size == 0) { *normsize = 0; return path; } From e28b0dc86dd1d058788b9e1eaa825cdfc2d97067 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 16 Aug 2023 02:04:17 +0800 Subject: [PATCH 141/598] gh-107557: Setup abstract interpretation (#107847) Co-authored-by: Guido van Rossum Co-authored-by: Jules <57632293+juliapoo@users.noreply.github.com> --- .gitattributes | 1 + Include/cpython/optimizer.h | 2 +- Include/internal/pycore_opcode_metadata.h | 142 ++++ Include/internal/pycore_optimizer.h | 20 + Include/internal/pycore_uops.h | 2 +- Makefile.pre.in | 12 +- ...-08-02-09-55-21.gh-issue-107557.P1z-in.rst | 1 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 3 + PCbuild/pythoncore.vcxproj.filters | 9 + Python/abstract_interp_cases.c.h | 761 ++++++++++++++++++ Python/bytecodes.c | 5 + Python/executor_cases.c.h | 9 + Python/optimizer.c | 17 +- Python/optimizer_analysis.c | 26 + Tools/c-analyzer/cpython/_parser.py | 1 + Tools/c-analyzer/cpython/ignored.tsv | 2 + Tools/cases_generator/generate_cases.py | 62 +- Tools/cases_generator/instructions.py | 19 + Tools/cases_generator/stacking.py | 29 + 21 files changed, 1118 insertions(+), 9 deletions(-) create mode 100644 Include/internal/pycore_optimizer.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst create mode 100644 Python/abstract_interp_cases.c.h create mode 100644 Python/optimizer_analysis.c diff --git a/.gitattributes b/.gitattributes index 4a072bc990f0fb..317a613217a9b3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -90,6 +90,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/abstract_interp_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index da34ec1882a539..e3fe0e8f0ffd59 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -22,7 +22,7 @@ typedef struct _PyExecutorObject { typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **); +typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); typedef struct _PyOptimizerObject { PyObject_HEAD diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c0a2e9d91c4f49..df9b4184e2a50e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -55,6 +55,7 @@ #define _POP_JUMP_IF_FALSE 331 #define _POP_JUMP_IF_TRUE 332 #define JUMP_TO_TOP 333 +#define INSERT 334 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -118,18 +119,38 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case UNARY_INVERT: return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; case BINARY_OP_MULTIPLY_INT: return 2; case BINARY_OP_ADD_INT: return 2; case BINARY_OP_SUBTRACT_INT: return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; case BINARY_OP_MULTIPLY_FLOAT: return 2; case BINARY_OP_ADD_FLOAT: return 2; case BINARY_OP_SUBTRACT_FLOAT: return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; case BINARY_OP_ADD_UNICODE: return 2; case BINARY_OP_INPLACE_ADD_UNICODE: @@ -226,14 +247,26 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case DELETE_GLOBAL: return 0; + case _LOAD_LOCALS: + return 0; case LOAD_LOCALS: return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case LOAD_NAME: return 0; case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_GLOBAL: return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; case LOAD_GLOBAL_MODULE: return 0; case LOAD_GLOBAL_BUILTIN: @@ -294,6 +327,12 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_METHOD: return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1; case LOAD_ATTR_MODULE: @@ -348,6 +387,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case POP_JUMP_IF_TRUE: return 1; + case IS_NONE: + return 1; case POP_JUMP_IF_NONE: return 1; case POP_JUMP_IF_NOT_NONE: @@ -372,10 +413,28 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case INSTRUMENTED_FOR_ITER: return 0; + case _ITER_CHECK_LIST: + return 1; + case _IS_ITER_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; case FOR_ITER_LIST: return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _IS_ITER_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; case FOR_ITER_TUPLE: return 1; + case _ITER_CHECK_RANGE: + return 1; + case _IS_ITER_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; case FOR_ITER_RANGE: return 1; case FOR_ITER_GEN: @@ -494,6 +553,18 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case RESERVED: return 0; + case _POP_JUMP_IF_FALSE: + return 1; + case _POP_JUMP_IF_TRUE: + return 1; + case JUMP_TO_TOP: + return 0; + case SAVE_IP: + return 0; + case EXIT_TRACE: + return 0; + case INSERT: + return oparg + 1; default: return -1; } @@ -562,18 +633,38 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case UNARY_INVERT: return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 1; + case _BINARY_OP_ADD_INT: + return 1; + case _BINARY_OP_SUBTRACT_INT: + return 1; case BINARY_OP_MULTIPLY_INT: return 1; case BINARY_OP_ADD_INT: return 1; case BINARY_OP_SUBTRACT_INT: return 1; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 1; + case _BINARY_OP_ADD_FLOAT: + return 1; + case _BINARY_OP_SUBTRACT_FLOAT: + return 1; case BINARY_OP_MULTIPLY_FLOAT: return 1; case BINARY_OP_ADD_FLOAT: return 1; case BINARY_OP_SUBTRACT_FLOAT: return 1; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 1; case BINARY_OP_ADD_UNICODE: return 1; case BINARY_OP_INPLACE_ADD_UNICODE: @@ -670,14 +761,26 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case DELETE_GLOBAL: return 0; + case _LOAD_LOCALS: + return 1; case LOAD_LOCALS: return 1; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case LOAD_NAME: return 1; case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_GLOBAL_BUILTINS: + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_GLOBAL_MODULE: return (oparg & 1 ? 1 : 0) + 1; case LOAD_GLOBAL_BUILTIN: @@ -738,6 +841,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case LOAD_METHOD: return ((oparg & 1) ? 1 : 0) + 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_MODULE: @@ -792,6 +901,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case POP_JUMP_IF_TRUE: return 0; + case IS_NONE: + return 1; case POP_JUMP_IF_NONE: return 0; case POP_JUMP_IF_NOT_NONE: @@ -816,10 +927,28 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case INSTRUMENTED_FOR_ITER: return 0; + case _ITER_CHECK_LIST: + return 1; + case _IS_ITER_EXHAUSTED_LIST: + return 2; + case _ITER_NEXT_LIST: + return 2; case FOR_ITER_LIST: return 2; + case _ITER_CHECK_TUPLE: + return 1; + case _IS_ITER_EXHAUSTED_TUPLE: + return 2; + case _ITER_NEXT_TUPLE: + return 2; case FOR_ITER_TUPLE: return 2; + case _ITER_CHECK_RANGE: + return 1; + case _IS_ITER_EXHAUSTED_RANGE: + return 2; + case _ITER_NEXT_RANGE: + return 2; case FOR_ITER_RANGE: return 2; case FOR_ITER_GEN: @@ -938,6 +1067,18 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RESERVED: return 0; + case _POP_JUMP_IF_FALSE: + return 0; + case _POP_JUMP_IF_TRUE: + return 0; + case JUMP_TO_TOP: + return 0; + case SAVE_IP: + return 0; + case EXIT_TRACE: + return 0; + case INSERT: + return oparg + 1; default: return -1; } @@ -1393,5 +1534,6 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [JUMP_TO_TOP] = "JUMP_TO_TOP", + [INSERT] = "INSERT", }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h new file mode 100644 index 00000000000000..2ae657c4e117ff --- /dev/null +++ b/Include/internal/pycore_optimizer.h @@ -0,0 +1,20 @@ +#ifndef Py_INTERNAL_OPTIMIZER_H +#define Py_INTERNAL_OPTIMIZER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_uops.h" + +int _Py_uop_analyze_and_optimize(PyCodeObject *code, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OPTIMIZER_H */ diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index 57a5970353b360..254eeca2361bea 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#define _Py_UOP_MAX_TRACE_LENGTH 32 +#define _Py_UOP_MAX_TRACE_LENGTH 64 typedef struct { uint32_t opcode; diff --git a/Makefile.pre.in b/Makefile.pre.in index 3a628bf49e97c1..32d928316f2215 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -405,6 +405,7 @@ PYTHON_OBJS= \ Python/mysnprintf.o \ Python/mystrtoul.o \ Python/optimizer.o \ + Python/optimizer_analysis.o \ Python/pathconfig.o \ Python/preconfig.o \ Python/pyarena.o \ @@ -1552,10 +1553,12 @@ regen-cases: -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ -e $(srcdir)/Python/executor_cases.c.h.new \ -p $(srcdir)/Lib/_opcode_metadata.py.new \ + -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Python/abstract_interp_cases.c.h $(srcdir)/Python/abstract_interp_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1568,6 +1571,7 @@ Python/ceval.o: \ Python/executor.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/executor_cases.c.h @@ -1576,7 +1580,12 @@ Python/flowgraph.o: \ Python/optimizer.o: \ $(srcdir)/Python/executor_cases.c.h \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_optimizer.h + +Python/optimizer_analysis.o: \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_optimizer.h Python/frozen.o: $(FROZEN_FILES_OUT) @@ -1786,6 +1795,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ $(srcdir)/Include/internal/pycore_opcode.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst new file mode 100644 index 00000000000000..392f59c79e8de9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst @@ -0,0 +1 @@ +Generate the cases needed for the barebones tier 2 abstract interpreter for optimization passes in CPython. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index e247637a0dfe5c..bdcf29ba44dab5 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -218,6 +218,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 2a0e009308022b..45333fa97f1c64 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -283,6 +283,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index bfe59acf12a69d..b0e62864421e17 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -248,6 +248,7 @@ + @@ -279,6 +280,7 @@ + @@ -548,6 +550,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0a8b0c3faf51e1..d5f61e9c5d7c89 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -648,6 +648,9 @@ Include\internal + + Include\internal + Include\internal @@ -732,6 +735,9 @@ Include\internal + + Include\internal + Modules\zlib @@ -1223,6 +1229,9 @@ Python + + Python + Python diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h new file mode 100644 index 00000000000000..6bfcf534646b1e --- /dev/null +++ b/Python/abstract_interp_cases.c.h @@ -0,0 +1,761 @@ +// This file is generated by Tools/cases_generator/generate_cases.py +// from: +// Python/bytecodes.c +// Do not edit! + + case NOP: { + break; + } + + case POP_TOP: { + STACK_SHRINK(1); + break; + } + + case PUSH_NULL: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case END_SEND: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case UNARY_NEGATIVE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case UNARY_NOT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_BOOL: { + break; + } + + case TO_BOOL_INT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_LIST: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_NONE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_STR: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_ALWAYS_TRUE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case UNARY_INVERT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _GUARD_BOTH_INT: { + break; + } + + case _GUARD_BOTH_FLOAT: { + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _BINARY_OP_ADD_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _GUARD_BOTH_UNICODE: { + break; + } + + case _BINARY_OP_ADD_UNICODE: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SLICE: { + STACK_SHRINK(2); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case STORE_SLICE: { + STACK_SHRINK(4); + break; + } + + case BINARY_SUBSCR_LIST_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR_STR_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR_TUPLE_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR_DICT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LIST_APPEND: { + STACK_SHRINK(1); + break; + } + + case SET_ADD: { + STACK_SHRINK(1); + break; + } + + case STORE_SUBSCR: { + STACK_SHRINK(3); + break; + } + + case STORE_SUBSCR_LIST_INT: { + STACK_SHRINK(3); + break; + } + + case STORE_SUBSCR_DICT: { + STACK_SHRINK(3); + break; + } + + case DELETE_SUBSCR: { + STACK_SHRINK(2); + break; + } + + case CALL_INTRINSIC_1: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_INTRINSIC_2: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_AITER: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_ANEXT: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_AWAITABLE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case POP_EXCEPT: { + STACK_SHRINK(1); + break; + } + + case LOAD_ASSERTION_ERROR: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_BUILD_CLASS: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case STORE_NAME: { + STACK_SHRINK(1); + break; + } + + case DELETE_NAME: { + break; + } + + case UNPACK_SEQUENCE: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_SEQUENCE_TWO_TUPLE: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_SEQUENCE_TUPLE: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_SEQUENCE_LIST: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_EX: { + STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg >> 8))), true); + break; + } + + case STORE_ATTR: { + STACK_SHRINK(2); + break; + } + + case DELETE_ATTR: { + STACK_SHRINK(1); + break; + } + + case STORE_GLOBAL: { + STACK_SHRINK(1); + break; + } + + case DELETE_GLOBAL: { + break; + } + + case _LOAD_LOCALS: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _LOAD_FROM_DICT_OR_GLOBALS: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_GLOBAL: { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _GUARD_GLOBALS_VERSION: { + break; + } + + case _GUARD_BUILTINS_VERSION: { + break; + } + + case _LOAD_GLOBAL_MODULE: { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case DELETE_FAST: { + break; + } + + case DELETE_DEREF: { + break; + } + + case LOAD_FROM_DICT_OR_DEREF: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_DEREF: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case STORE_DEREF: { + STACK_SHRINK(1); + break; + } + + case COPY_FREE_VARS: { + break; + } + + case BUILD_STRING: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_TUPLE: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_LIST: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LIST_EXTEND: { + STACK_SHRINK(1); + break; + } + + case SET_UPDATE: { + STACK_SHRINK(1); + break; + } + + case BUILD_SET: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_MAP: { + STACK_SHRINK(oparg*2); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case SETUP_ANNOTATIONS: { + break; + } + + case BUILD_CONST_KEY_MAP: { + STACK_SHRINK(oparg); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case DICT_UPDATE: { + STACK_SHRINK(1); + break; + } + + case DICT_MERGE: { + STACK_SHRINK(1); + break; + } + + case MAP_ADD: { + STACK_SHRINK(2); + break; + } + + case LOAD_SUPER_ATTR_ATTR: { + STACK_SHRINK(2); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); + break; + } + + case LOAD_SUPER_ATTR_METHOD: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_ATTR: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _GUARD_TYPE_VERSION: { + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case COMPARE_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case COMPARE_OP_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case COMPARE_OP_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case COMPARE_OP_STR: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case IS_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CONTAINS_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CHECK_EG_MATCH: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CHECK_EXC_MATCH: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case IS_NONE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_LEN: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_CLASS: { + STACK_SHRINK(2); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_MAPPING: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_SEQUENCE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_KEYS: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_ITER: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_YIELD_FROM_ITER: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_CHECK_LIST: { + break; + } + + case _IS_ITER_EXHAUSTED_LIST: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_NEXT_LIST: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_CHECK_TUPLE: { + break; + } + + case _IS_ITER_EXHAUSTED_TUPLE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_NEXT_TUPLE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_CHECK_RANGE: { + break; + } + + case _IS_ITER_EXHAUSTED_RANGE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_NEXT_RANGE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case WITH_EXCEPT_START: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case PUSH_EXC_INFO: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_TYPE_1: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_STR_1: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_TUPLE_1: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case EXIT_INIT_CHECK: { + STACK_SHRINK(1); + break; + } + + case CALL_NO_KW_BUILTIN_O: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_BUILTIN_FAST: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_LEN: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_ISINSTANCE: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_O: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MAKE_FUNCTION: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case SET_FUNCTION_ATTRIBUTE: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_SLICE: { + STACK_SHRINK(((oparg == 3) ? 1 : 0)); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CONVERT_VALUE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case FORMAT_SIMPLE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case FORMAT_WITH_SPEC: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case SWAP: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - (oparg-2))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _POP_JUMP_IF_FALSE: { + STACK_SHRINK(1); + break; + } + + case _POP_JUMP_IF_TRUE: { + STACK_SHRINK(1); + break; + } + + case JUMP_TO_TOP: { + break; + } + + case SAVE_IP: { + break; + } + + case EXIT_TRACE: { + break; + } + + case INSERT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); + break; + } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5efa36fcf5c629..e9a5cf59e7d689 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3743,6 +3743,11 @@ dummy_func( return frame; } + op(INSERT, (unused[oparg], top -- top, unused[oparg])) { + // Inserts TOS at position specified by oparg; + memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + } + // END BYTECODES // diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5e3c84b2b4b612..85d27777423abd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2733,3 +2733,12 @@ return frame; break; } + + case INSERT: { + PyObject *top; + top = stack_pointer[-1]; + // Inserts TOS at position specified by oparg; + memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + stack_pointer[-1 - oparg] = top; + break; + } diff --git a/Python/optimizer.c b/Python/optimizer.c index 6c730aa14b9a47..d3ac2424038ef9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -4,6 +4,7 @@ #include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uops.h" #include "cpython/optimizer.h" @@ -103,7 +104,8 @@ error_optimize( _PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, - _PyExecutorObject **exec) + _PyExecutorObject **exec, + int Py_UNUSED(stack_entries)) { PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); return -1; @@ -164,7 +166,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI } _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, code, dest, &executor); + int err = opt->optimize(opt, code, dest, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { assert(executor == NULL); if (err < 0) { @@ -254,7 +256,9 @@ counter_optimize( _PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr) + _PyExecutorObject **exec_ptr, + int Py_UNUSED(curr_stackentries) +) { _PyCounterExecutorObject *executor = (_PyCounterExecutorObject *)_PyObject_New(&CounterExecutor_Type); if (executor == NULL) { @@ -684,7 +688,8 @@ uop_optimize( _PyOptimizerObject *self, PyCodeObject *code, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr) + _PyExecutorObject **exec_ptr, + int curr_stackentries) { _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; int trace_length = translate_bytecode_to_trace(code, instr, trace, _Py_UOP_MAX_TRACE_LENGTH); @@ -693,6 +698,10 @@ uop_optimize( return trace_length; } OBJECT_STAT_INC(optimization_traces_created); + char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); + if (uop_optimize != NULL && *uop_optimize > '0') { + trace_length = _Py_uop_analyze_and_optimize(code, trace, trace_length, curr_stackentries); + } _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); if (executor == NULL) { return -1; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c new file mode 100644 index 00000000000000..e48e018052c712 --- /dev/null +++ b/Python/optimizer_analysis.c @@ -0,0 +1,26 @@ +#include "Python.h" +#include "opcode.h" +#include "pycore_interp.h" +#include "pycore_opcode.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_uops.h" +#include "pycore_long.h" +#include "cpython/optimizer.h" +#include +#include +#include +#include "pycore_optimizer.h" + + +int +_Py_uop_analyze_and_optimize( + PyCodeObject *co, + _PyUOpInstruction *trace, + int trace_len, + int curr_stacklen +) +{ + return trace_len; +} diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 9bc7285e18b2fb..90334d0e79da80 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -84,6 +84,7 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h +Python/abstract_interp_cases.c.h # not actually source Python/bytecodes.c diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c64d391bae13bd..d1ac0410619c96 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -716,3 +716,5 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - +Python/optimizer_analysis.c - _Py_PartitionRootNode_Type - +Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index e78b0edff9a0c8..e170e110f80cfb 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -16,6 +16,7 @@ from flags import InstructionFlags, variable_used from instructions import ( AnyInstruction, + AbstractInstruction, Component, Instruction, MacroInstruction, @@ -44,6 +45,9 @@ DEFAULT_EXECUTOR_OUTPUT = os.path.relpath( os.path.join(ROOT, "Python/executor_cases.c.h") ) +DEFAULT_ABSTRACT_INTERPRETER_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Python/abstract_interp_cases.c.h") +) # Constants used instead of size for macro expansions. # Note: 1, 2, 4 must match actual cache entry sizes. @@ -58,6 +62,23 @@ INSTR_FMT_PREFIX = "INSTR_FMT_" +# TODO: generate all these after updating the DSL +SPECIALLY_HANDLED_ABSTRACT_INSTR = { + "LOAD_FAST", + "LOAD_FAST_CHECK", + "LOAD_FAST_AND_CLEAR", + "LOAD_CONST", + "STORE_FAST", + "STORE_FAST_MAYBE_NULL", + "COPY", + + # Arithmetic + "_BINARY_OP_MULTIPLY_INT", + "_BINARY_OP_ADD_INT", + "_BINARY_OP_SUBTRACT_INT", + +} + arg_parser = argparse.ArgumentParser( description="Generate the code for the interpreter switch.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -92,7 +113,13 @@ help="Write executor cases to this file", default=DEFAULT_EXECUTOR_OUTPUT, ) - +arg_parser.add_argument( + "-a", + "--abstract-interpreter-cases", + type=str, + help="Write abstract interpreter cases to this file", + default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, +) class Generator(Analyzer): def get_stack_effect_info( @@ -109,7 +136,7 @@ def effect_str(effects: list[StackEffect]) -> str: pushed: str | None match thing: case parsing.InstDef(): - if thing.kind != "op": + if thing.kind != "op" or self.instrs[thing.name].is_viable_uop(): instr = self.instrs[thing.name] popped = effect_str(instr.input_effects) pushed = effect_str(instr.output_effects) @@ -588,6 +615,35 @@ def write_executor_instructions( file=sys.stderr, ) + def write_abstract_interpreter_instructions( + self, abstract_interpreter_filename: str, emit_line_directives: bool + ) -> None: + """Generate cases for the Tier 2 abstract interpreter/analzyer.""" + with open(abstract_interpreter_filename, "w") as f: + self.out = Formatter(f, 8, emit_line_directives) + self.write_provenance_header() + for thing in self.everything: + match thing: + case OverriddenInstructionPlaceHolder(): + pass + case parsing.InstDef(): + instr = AbstractInstruction(self.instrs[thing.name].inst) + if instr.is_viable_uop() and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR: + self.out.emit("") + with self.out.block(f"case {thing.name}:"): + instr.write(self.out, tier=TIER_TWO) + self.out.emit("break;") + case parsing.Macro(): + pass + case parsing.Pseudo(): + pass + case _: + typing.assert_never(thing) + print( + f"Wrote some stuff to {abstract_interpreter_filename}", + file=sys.stderr, + ) + def write_overridden_instr_place_holder( self, place_holder: OverriddenInstructionPlaceHolder ) -> None: @@ -629,6 +685,8 @@ def main(): a.write_instructions(args.output, args.emit_line_directives) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) + a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, + args.emit_line_directives) if __name__ == "__main__": diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index aa94dbb07ea1c0..a505df08fa265b 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -248,6 +248,25 @@ def write_body( InstructionOrCacheEffect = Instruction | parsing.CacheEffect +# Instruction used for abstract interpretation. +class AbstractInstruction(Instruction): + def __init__(self, inst: parsing.InstDef): + super().__init__(inst) + + def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: + """Write one abstract instruction, sans prologue and epilogue.""" + stacking.write_single_instr_for_abstract_interp(self, out) + + def write_body( + self, + out: Formatter, + dedent: int, + active_caches: list[ActiveCacheEffect], + tier: Tiers = TIER_ONE, + ) -> None: + pass + + @dataclasses.dataclass class Component: instr: Instruction diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 9bb7f468442245..31a21e026cb49c 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -391,3 +391,32 @@ def write_components( poke.as_stack_effect(), poke.effect, ) + + +def write_single_instr_for_abstract_interp( + instr: Instruction, out: Formatter +): + try: + _write_components_for_abstract_interp( + [Component(instr, instr.active_caches)], + out, + ) + except AssertionError as err: + raise AssertionError(f"Error writing abstract instruction {instr.name}") from err + + +def _write_components_for_abstract_interp( + parts: list[Component], + out: Formatter, +): + managers = get_managers(parts) + for mgr in managers: + if mgr is managers[-1]: + out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + # NULL out the output stack effects + for poke in mgr.pokes: + if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.emit(f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" + f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);") From 99b6ce56f8185bb936cae3b2b67ae24b11853e30 Mon Sep 17 00:00:00 2001 From: Martin DeMello Date: Tue, 15 Aug 2023 13:27:35 -0700 Subject: [PATCH 142/598] Note that lnotab_notes.txt is only valid before 3.11 (#107961) --- Objects/lnotab_notes.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 362b87a86a481f..d45d09d4ab9a50 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,4 +1,7 @@ -Description of the internal format of the line number table +Description of the internal format of the line number table in Python 3.10 +and earlier. + +(For 3.11 onwards, see Objects/locations.md) Conceptually, the line number table consists of a sequence of triples: start-offset (inclusive), end-offset (exclusive), line-number. From bb456a08a3db851e6feaefc3328f39096919ec8d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 15 Aug 2023 22:45:53 +0200 Subject: [PATCH 143/598] gh-106368: Argument Clinic: Add tests for cloned functions with custom C base names (#107977) --- Lib/test/test_clinic.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 896730571d8152..76729c8c3e4f79 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -653,6 +653,37 @@ def test_no_c_basename_cloned(self): err = "No C basename provided after 'as' keyword" self.expect_failure(block, err, lineno=5) + def test_cloned_with_custom_c_basename(self): + raw = dedent(""" + /*[clinic input] + # Make sure we don't create spurious clinic/ directories. + output everything suppress + foo2 + [clinic start generated code]*/ + + /*[clinic input] + foo as foo1 = foo2 + [clinic start generated code]*/ + """) + self.clinic.parse(raw) + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + self.assertEqual(funcs[1].name, "foo") + self.assertEqual(funcs[1].c_basename, "foo1") + + def test_cloned_with_illegal_c_basename(self): + block = """ + /*[clinic input] + class C "void *" "" + foo1 + [clinic start generated code]*/ + + /*[clinic input] + foo2 as .illegal. = foo1 + [clinic start generated code]*/ + """ + err = "Illegal C basename: '.illegal. = foo1'" + self.expect_failure(block, err, lineno=7) class ParseFileUnitTest(TestCase): From a794ebeb028f7ef287c780d3890f816db9c21c51 Mon Sep 17 00:00:00 2001 From: Yuxin Wu Date: Tue, 15 Aug 2023 18:03:45 -0700 Subject: [PATCH 144/598] More actionable error message when spawn is incorrectly used. (#102203) Co-authored-by: Yuxin Wu Co-authored-by: Oleg Iarygin --- Lib/multiprocessing/spawn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index f1af7709104714..daac1ecc34b55e 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -150,7 +150,11 @@ def _check_not_importing_main(): ... The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') + is not going to be frozen to produce an executable. + + To fix this issue, refer to the "Safe importing of main module" + section in https://docs.python.org/3/library/multiprocessing.html + ''') def get_preparation_data(name): From a86df298df5b02e2d69ea6879e9ed10a7adb85d0 Mon Sep 17 00:00:00 2001 From: 6t8k <58048945+6t8k@users.noreply.github.com> Date: Wed, 16 Aug 2023 07:00:03 +0000 Subject: [PATCH 145/598] gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) Restore following CPython <= 3.10.5 behavior of shutil.make_archive() that went away as part of gh-93160: Do not create an empty archive if root_dir is not a directory, and, in that case, raise FileNotFoundError or NotADirectoryError regardless of format choice. Beyond the brought-back behavior, the function may now also raise these exceptions in dry_run mode. --- Lib/shutil.py | 4 ++ Lib/test/test_shutil.py | 43 +++++++++++++++++++ ...2-11-26-22-05-22.gh-issue-99203.j0DUae.rst | 5 +++ 3 files changed, 52 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index 3f2864af517e7d..b37bd082eee0c6 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1156,6 +1156,10 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, supports_root_dir = getattr(func, 'supports_root_dir', False) save_cwd = None if root_dir is not None: + stmd = os.stat(root_dir).st_mode + if not stat.S_ISDIR(stmd): + raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir) + if supports_root_dir: # Support path-like base_name here for backwards-compatibility. base_name = os.fspath(base_name) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 93f20a6ff41332..a2ca4df135846f 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1839,6 +1839,49 @@ def test_register_archive_format(self): formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) + def test_make_tarfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'tar', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'tar', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + @support.requires_zlib() + def test_make_zipfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'zip', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'zip', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + ### shutil.unpack_archive def check_unpack_archive(self, format, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst new file mode 100644 index 00000000000000..fcfb044d476acc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst @@ -0,0 +1,5 @@ +Restore following CPython <= 3.10.5 behavior of :func:`shutil.make_archive`: +do not create an empty archive if ``root_dir`` is not a directory, and, in that +case, raise :class:`FileNotFoundError` or :class:`NotADirectoryError` +regardless of ``format`` choice. Beyond the brought-back behavior, the function +may now also raise these exceptions in ``dry_run`` mode. From abd9cc52d94b8e2835322b62c29f09bb0e6fcfe9 Mon Sep 17 00:00:00 2001 From: SKO <41810398+uyw4687@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:43:45 +0900 Subject: [PATCH 146/598] gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) Restore the global Input Stream pointer after trying to match a sub-pattern. Co-authored-by: Ma Lin --- Lib/re/_compiler.py | 7 ------- Lib/test/test_re.py | 14 +++++++++++--- .../2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst | 2 ++ Modules/_sre/sre_lib.h | 4 ++++ 4 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index f5fd160ba00435..d0a4c55caf6e41 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,13 +100,6 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) - elif op is POSSESSIVE_REPEAT: - # gh-106052: Possessive quantifiers do not work when the - # subpattern contains backtracking, i.e. "(?:ab?c)*+". - # Implement it as equivalent greedy qualifier in atomic group. - p = [(MAX_REPEAT, av)] - p = [(ATOMIC_GROUP, p)] - _compile(code, p, flags) elif op in REPEATING_CODES: if _simple(av[2]): emit(REPEATING_CODES[op][2]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index bf3698ac78a880..042f97f57ecf18 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2342,7 +2342,17 @@ def test_bug_gh91616(self): self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer self.assertTrue(re.fullmatch(r'(?s:(?=(?P.*?\.))(?P=g0).*)\Z', "a.txt")) - def test_bug_gh106052(self): + def test_bug_gh100061(self): + # gh-100061 + self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))++', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))*)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))*+', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))?)', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?:.(?!D))?+', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?>(?:.(?!D)){1,3})', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D)){1,3}+', 'ABCDE').span(), (0, 2)) + # gh-106052 self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) @@ -2451,7 +2461,6 @@ def test_atomic_group(self): 17: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2464,7 +2473,6 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst new file mode 100644 index 00000000000000..dfed34f6ae9768 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst @@ -0,0 +1,2 @@ +Fix a bug that causes wrong matches for regular expressions with possessive +qualifier. diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index c1a774f69090b3..ae80009fd63bbe 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -1336,6 +1336,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); + /* Restore the global Input Stream pointer + since it can change after jumps. */ + state->ptr = ptr; + /* We have sufficient matches, so exit loop. */ break; } From f6099871fac0581ed4d9bcd9b15ce136fb0de8d6 Mon Sep 17 00:00:00 2001 From: brandonardenwalli <142186236+brandonardenwalli@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:14:14 +0200 Subject: [PATCH 147/598] gh-107955 Remove old comment about increasing the reference count in usage of Py_None (#107993) --- Include/object.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Include/object.h b/Include/object.h index e26cedf8ca3c97..be9a0cedb7205d 100644 --- a/Include/object.h +++ b/Include/object.h @@ -845,8 +845,6 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) /* _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). - -Don't forget to apply Py_INCREF() when returning this value!!! */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) From fd9d70a94de5b0756b52b9ae21e236e25545db4f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 16 Aug 2023 12:20:42 +0300 Subject: [PATCH 148/598] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) Use a custom exception to prevent unintentional silence of actual errors. --- Lib/test/test_unittest/test_runner.py | 117 ++++++++++++++------------ 1 file changed, 65 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index f3b2c0cffd4513..1b9cef43e3f9c5 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -24,6 +24,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -46,7 +53,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -108,8 +115,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -125,10 +132,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -139,7 +146,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -280,7 +287,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -306,7 +313,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -346,7 +353,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -366,10 +373,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -379,7 +386,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -392,16 +399,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -421,7 +434,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -444,7 +457,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -460,11 +473,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -473,7 +486,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -483,9 +496,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -494,9 +507,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -513,11 +526,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -620,7 +633,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -630,7 +643,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -659,7 +672,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -679,7 +692,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -699,7 +712,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -710,7 +723,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -799,7 +812,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -815,7 +828,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -855,7 +868,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -873,7 +886,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -888,7 +901,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -899,7 +912,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -978,7 +991,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1008,7 +1021,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1024,7 +1037,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1034,11 +1047,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1050,7 +1063,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1062,9 +1075,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1073,9 +1086,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1085,9 +1098,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) From bdd8ddfda166d1ed49744d61dcc486d62a9ac890 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 16 Aug 2023 13:35:38 +0300 Subject: [PATCH 149/598] gh-105724: Add location information to `assert` errors (GH-105935) --- Lib/test/test_compile.py | 10 +- Lib/test/test_exceptions.py | 117 ++++++++++++++++++ ...-06-20-10-53-17.gh-issue-105724.d23L4M.rst | 1 + Python/compile.c | 2 +- 4 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 770184c5ef9a91..2ed8ae04961376 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1315,18 +1315,18 @@ def test_multiline_assert(self): snippet = textwrap.dedent("""\ assert (a > 0 and bb > 0 and - ccc == 4), "error msg" + ccc == 1000000), "error msg" """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', - line=1, end_line=3, column=0, end_column=30, occurrence=1) + line=1, end_line=3, column=0, end_column=36, occurrence=1) # The "error msg": self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', - line=3, end_line=3, column=19, end_column=30, occurrence=4) + line=3, end_line=3, column=25, end_column=36, occurrence=4) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', - line=1, end_line=3, column=0, end_column=30, occurrence=1) + line=1, end_line=3, column=0, end_column=36, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', - line=1, end_line=3, column=0, end_column=30, occurrence=1) + line=1, end_line=3, column=8, end_column=22, occurrence=1) def test_multiline_generator_expression(self): snippet = textwrap.dedent("""\ diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index f3554f1c4bb3f3..764122ed4ef783 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1931,6 +1931,123 @@ def test_copy_pickle(self): self.assertEqual(exc.name, orig.name) self.assertEqual(exc.path, orig.path) + +class AssertionErrorTests(unittest.TestCase): + def tearDown(self): + unlink(TESTFN) + + def write_source(self, source): + with open(TESTFN, 'w') as testfile: + testfile.write(dedent(source)) + _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) + return err.decode('utf-8').splitlines() + + def test_assertion_error_location(self): + cases = [ + ('assert None', + [ + ' assert None', + ' ^^^^', + 'AssertionError', + ], + ), + ('assert 0', + [ + ' assert 0', + ' ^', + 'AssertionError', + ], + ), + ('assert 1 > 2', + [ + ' assert 1 > 2', + ' ^^^^^', + 'AssertionError', + ], + ), + ('assert 1 > 2 and 3 > 2', + [ + ' assert 1 > 2 and 3 > 2', + ' ^^^^^^^^^^^^^^^', + 'AssertionError', + ], + ), + ('assert 1 > 2, "message"', + [ + ' assert 1 > 2, "message"', + ' ^^^^^', + 'AssertionError: message', + ], + ), + + # Multiline: + (""" + assert ( + 1 > 2) + """, + [ + ' 1 > 2)', + ' ^^^^^', + 'AssertionError', + ], + ), + (""" + assert ( + 1 > 2), "Message" + """, + [ + ' 1 > 2), "Message"', + ' ^^^^^', + 'AssertionError: Message', + ], + ), + (""" + assert ( + 1 > 2), \\ + "Message" + """, + [ + ' 1 > 2), \\', + ' ^^^^^', + 'AssertionError: Message', + ], + ), + ] + for source, expected in cases: + with self.subTest(source): + result = self.write_source(source) + self.assertEqual(result[-3:], expected) + + def test_multiline_not_highlighted(self): + cases = [ + (""" + assert ( + 1 > 2 + ) + """, + [ + ' 1 > 2', + 'AssertionError', + ], + ), + (""" + assert ( + 1 < 2 and + 3 > 4 + ) + """, + [ + ' 1 < 2 and', + 'AssertionError', + ], + ), + ] + for source, expected in cases: + with self.subTest(source): + result = self.write_source(source) + self.assertEqual(result[-2:], expected) + + class SyntaxErrorTests(unittest.TestCase): def test_range_of_offsets(self): cases = [ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst new file mode 100644 index 00000000000000..281c139d1a8d1c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst @@ -0,0 +1 @@ +Improve ``assert`` error messages by providing exact error range. diff --git a/Python/compile.c b/Python/compile.c index 83cf45550e2588..3260dba57eac8f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3952,7 +3952,7 @@ compiler_assert(struct compiler *c, stmt_ty s) VISIT(c, expr, s->v.Assert.msg); ADDOP_I(c, LOC(s), CALL, 0); } - ADDOP_I(c, LOC(s), RAISE_VARARGS, 1); + ADDOP_I(c, LOC(s->v.Assert.test), RAISE_VARARGS, 1); USE_LABEL(c, end); return SUCCESS; From 57a20b0960f5c087a476b34c72f608580746cab5 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 16 Aug 2023 12:39:56 +0200 Subject: [PATCH 150/598] gh-106368: Argument Clinic: Test that keyword params are disallowed in groups (#107985) --- Lib/test/test_clinic.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 76729c8c3e4f79..32aac407a028c9 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1524,6 +1524,27 @@ def test_disallowed_grouping__no_matching_bracket(self): err = "Function 'empty_group' has a ']' without a matching '['" self.expect_failure(block, err) + def test_disallowed_grouping__must_be_position_only(self): + dataset = (""" + with_kwds + [ + * + a: object + ] + """, """ + with_kwds + [ + a: object + ] + """) + err = ( + "You cannot use optional groups ('[' and ']') unless all " + "parameters are positional-only ('/')" + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + def test_no_parameters(self): function = self.parse_function(""" module foo From e35c722d22cae605b485e75a69238dc44aab4c96 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Aug 2023 13:56:56 +0200 Subject: [PATCH 151/598] gh-106659: Fix test_embed.test_forced_io_encoding() on Windows (#108010) Use config.legacy_windows_stdio=1 to avoid _io._WindowsConsoleIO. --- Doc/whatsnew/3.13.rst | 4 +++- Programs/_testembed.c | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a65a98bae13579..84ffd84b9f0bb6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -941,7 +941,9 @@ Removed * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. - * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` instead. + * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` + instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on + Windows). * ``_Py_SetProgramFullPath()``: set :c:member:`PyConfig.executable` instead. Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 351cdc302e8cb0..7ee64b22925f0c 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -237,6 +237,11 @@ static void check_stdio_details(const wchar_t *encoding, const wchar_t *errors) if (errors) { config_set_string(&config, &config.stdio_errors, errors); } +#ifdef MS_WINDOWS + // gh-106659: On Windows, don't use _io._WindowsConsoleIO which always + // announce UTF-8 for sys.stdin.encoding. + config.legacy_windows_stdio = 1; +#endif config_set_program_name(&config); init_from_config_clear(&config); From a8d440b3837273926af5ce996162b019290ddad5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 16 Aug 2023 16:22:18 +0300 Subject: [PATCH 152/598] gh-108000: Test that `lambda` also has `__type_params__` (#108002) --- Lib/test/test_funcattrs.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index e08d72877d8aef..35b473d5e9a0b4 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -194,16 +194,19 @@ def test___qualname__(self): def test___type_params__(self): def generic[T](): pass def not_generic(): pass + lambda_ = lambda: ... T, = generic.__type_params__ self.assertIsInstance(T, typing.TypeVar) self.assertEqual(generic.__type_params__, (T,)) - self.assertEqual(not_generic.__type_params__, ()) - with self.assertRaises(TypeError): - del not_generic.__type_params__ - with self.assertRaises(TypeError): - not_generic.__type_params__ = 42 - not_generic.__type_params__ = (T,) - self.assertEqual(not_generic.__type_params__, (T,)) + for func in (not_generic, lambda_): + with self.subTest(func=func): + self.assertEqual(func.__type_params__, ()) + with self.assertRaises(TypeError): + del func.__type_params__ + with self.assertRaises(TypeError): + func.__type_params__ = 42 + func.__type_params__ = (T,) + self.assertEqual(func.__type_params__, (T,)) def test___code__(self): num_one, num_two = 7, 8 From b61f5995aebb93496e968ca8d307375fa86d9329 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 16 Aug 2023 16:30:03 +0300 Subject: [PATCH 153/598] gh-107909: Test explicit `object` base in PEP695 generic classes (#108001) --- Lib/test/test_type_params.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index bced641a9661fd..0045057f181e1c 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -148,6 +148,10 @@ def test_disallowed_expressions(self): check_syntax_error(self, "def f[T: [(x := 3) for _ in range(2)]](): pass") check_syntax_error(self, "type T = [(x := 3) for _ in range(2)]") + def test_incorrect_mro_explicit_object(self): + with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): + class My[X](object): ... + class TypeParamsNonlocalTest(unittest.TestCase): def test_nonlocal_disallowed_01(self): From 42429d3b9adb8af1eadcfa155f6e8422a254ec67 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 16 Aug 2023 15:47:15 +0200 Subject: [PATCH 154/598] gh-104683: Argument Clinic: Extract parse function name helper (#107964) Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 2 +- Tools/clinic/clinic.py | 61 ++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 32aac407a028c9..38eabd2f7f21dc 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -682,7 +682,7 @@ class C "void *" "" foo2 as .illegal. = foo1 [clinic start generated code]*/ """ - err = "Illegal C basename: '.illegal. = foo1'" + err = "Illegal C basename: '.illegal.'" self.expect_failure(block, err, lineno=7) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6ff2622d33b381..9f7c47430772f7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1674,18 +1674,8 @@ def render_function( full_name = f.full_name template_dict = {'full_name': full_name} template_dict['name'] = f.displayname - - if f.c_basename: - c_basename = f.c_basename - else: - fields = full_name.split(".") - if fields[-1] == '__new__': - fields.pop() - c_basename = "_".join(fields) - - template_dict['c_basename'] = c_basename - - template_dict['methoddef_name'] = c_basename.upper() + "_METHODDEF" + template_dict['c_basename'] = f.c_basename + template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" template_dict['docstring'] = self.docstring_for_c_string(f) @@ -2653,7 +2643,7 @@ class Function: name: str module: Module | Clinic cls: Class | None - c_basename: str | None + c_basename: str full_name: str return_converter: CReturnConverter kind: FunctionKind @@ -4577,6 +4567,11 @@ class ParamState(enum.IntEnum): RIGHT_SQUARE_AFTER = 6 +class FunctionNames(NamedTuple): + full_name: str + c_basename: str + + class DSLParser: function: Function | None state: StateKeeper @@ -4840,6 +4835,24 @@ def state_dsl_start(self, line: str) -> None: self.next(self.state_modulename_name, line) + @staticmethod + def parse_function_names(line: str) -> FunctionNames: + left, as_, right = line.partition(' as ') + full_name = left.strip() + c_basename = right.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") + if not c_basename: + fields = full_name.split(".") + if fields[-1] == '__new__': + fields.pop() + c_basename = "_".join(fields) + if not is_legal_py_identifier(full_name): + fail(f"Illegal function name: {full_name!r}") + if not is_legal_c_identifier(c_basename): + fail(f"Illegal C basename: {c_basename!r}") + return FunctionNames(full_name=full_name, c_basename=c_basename) + def update_function_kind(self, fullname: str) -> None: fields = fullname.split('.') name = fields.pop() @@ -4877,17 +4890,10 @@ def state_modulename_name(self, line: str) -> None: # are we cloning? before, equals, existing = line.rpartition('=') - c_basename: str | None if equals: - full_name, as_, c_basename = before.partition(' as ') - full_name = full_name.strip() - c_basename = c_basename.strip() - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") + full_name, c_basename = self.parse_function_names(before) existing = existing.strip() - if (is_legal_py_identifier(full_name) and - (not c_basename or is_legal_c_identifier(c_basename)) and - is_legal_py_identifier(existing)): + if is_legal_py_identifier(existing): # we're cloning! fields = [x.strip() for x in existing.split('.')] function_name = fields.pop() @@ -4933,16 +4939,7 @@ def state_modulename_name(self, line: str) -> None: line, _, returns = line.partition('->') returns = returns.strip() - - full_name, as_, c_basename = line.partition(' as ') - full_name = full_name.strip() - c_basename = c_basename.strip() or None - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") - if not is_legal_py_identifier(full_name): - fail(f"Illegal function name: {full_name!r}") - if c_basename and not is_legal_c_identifier(c_basename): - fail(f"Illegal C basename: {c_basename!r}") + full_name, c_basename = self.parse_function_names(line) return_converter = None if returns: From f2a9dfdee9de381e4adf29a7f1e2aec56580bfda Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:10:44 +0100 Subject: [PATCH 155/598] GH-92584: Drop reference to Distutils in ``site.USER_BASE`` (#108031) Drop reference to Distutils in ``site.USER_BASE`` --- Doc/library/site.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 44f90a3b9e496f..ff9e9107f9d163 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -189,7 +189,7 @@ Module contents :func:`getuserbase` hasn't been called yet. Default value is :file:`~/.local` for UNIX and macOS non-framework builds, :file:`~/Library/Python/{X.Y}` for macOS framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to + :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, etc. for the :ref:`user installation scheme `. See also :envvar:`PYTHONUSERBASE`. From c2941cba7a986e6158eebb2a0bf33906dcd78616 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Aug 2023 18:24:46 +0200 Subject: [PATCH 156/598] gh-107298: Fix C API Buffer documentation (#108011) --- Doc/c-api/buffer.rst | 22 +++++++++++++--------- Doc/c-api/typeobj.rst | 2 +- Doc/tools/.nitignore | 1 - 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index ba391a5279f205..e572815ffd6259 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -163,13 +163,6 @@ a buffer, see :c:func:`PyObject_GetBuffer`. and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - .. :c:macro:: PyBUF_MAX_NDIM - - The maximum number of dimensions the memory represents. - Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. - Currently set to 64. - .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` @@ -221,6 +214,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`. freed when the buffer is released. The consumer MUST NOT alter this value. + +Constants: + +.. c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. + + .. _buffer-request-types: Buffer request types @@ -444,7 +448,7 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and + :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. On success, fill in *view*, set ``view->obj`` to a new reference @@ -531,7 +535,7 @@ Buffer-related functions and :c:macro:`PyBUF_WRITABLE` is set in *flags*. On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set + return 0. Otherwise, raise :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index faa183e27fcfa2..d394ce10504b0e 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2443,7 +2443,7 @@ Buffer Object Structures Except for point (3), an implementation of this function MUST take these steps: - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, + (1) Check if the request can be met. If not, raise :exc:`BufferError`, set :c:expr:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 21b350c4134fd7..0f68cefc92b97b 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/buffer.rst Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst From 0d7f5d3ba3641f8c7d32facbb177bf70ee7520d1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Aug 2023 21:18:56 +0300 Subject: [PATCH 157/598] Remove Sphinx problem matcher to avoid annotating unchanged files (#108005) --- .github/problem-matchers/sphinx.json | 40 ---------------------------- .github/workflows/reusable-docs.yml | 4 --- 2 files changed, 44 deletions(-) delete mode 100644 .github/problem-matchers/sphinx.json diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json deleted file mode 100644 index 09848608a7b034..00000000000000 --- a/.github/problem-matchers/sphinx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "sphinx-problem-matcher", - "pattern": [ - { - "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "line": 2, - "severity": 3, - "message": 4 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose", - "pattern": [ - { - "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", - "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "severity": 2, - "message": 3 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose-no-severity", - "pattern": [ - { - "_comment": "Looks for file names ending with .rst and line numbers but without severity", - "regexp": "^(.*\\.rst):(\\d+):(.*)$", - "file": 1, - "line": 2, - "message": 3 - } - ] - } - ] -} \ No newline at end of file diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 56932c4860573c..39e5ad62924ad3 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -18,8 +18,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - name: 'Set up Python' uses: actions/setup-python@v4 with: @@ -76,8 +74,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - uses: actions/cache@v3 with: path: ~/.cache/pip From 8891a8821d5b03cd83a126fd6c02649448b18f41 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 16 Aug 2023 19:38:58 +0100 Subject: [PATCH 158/598] Improve the feature-proposal issue form (#108033) Co-authored-by: Hugo van Kemenade Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/feature.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index a1c48bbff829c4..0200e623d2a3b0 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -10,28 +10,26 @@ body: You'll need to demonstrate widespread support for your idea among the community. Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. - - type: checkboxes + - type: dropdown attributes: - label: Has this already been discussed elsewhere? + label: Has this already been discussed elsewhere? options: - - label: I have already discussed this feature proposal on Discourse - - label: This is a minor feature, which does not need previous discussion elsewhere + - No response given + - I have already discussed this feature proposal on Discourse + - This is a minor feature, which does not need previous discussion elsewhere + multiple: false + validations: + required: true - type: textarea attributes: label: "Links to previous discussion of this feature:" validations: required: false - - type: input - attributes: - label: "Summary of proposal:" - description: A one-line summary of your proposal. - validations: - required: true - type: textarea attributes: - label: "Pitch:" + label: "Proposal:" description: > - Explain why this feature or enhancement should be implemented and how it would be used. + Explain your proposal, why it should be implemented, and how it would be used. Add examples, if applicable. Put any code blocks inside triple backticks. value: | From fce93c80ae2d792b8ca443b044e28abbf28bb89a Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 16 Aug 2023 13:13:32 -0600 Subject: [PATCH 159/598] gh-91051: fix type watcher test to be robust to existing watcher (#107989) --- Lib/test/test_capi/test_watchers.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 10b76e163bfb21..6b8855ec219d27 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -351,12 +351,10 @@ def test_clear_unassigned_watcher_id(self): self.clear_watcher(1) def test_no_more_ids_available(self): - contexts = [self.watcher() for i in range(self.TYPE_MAX_WATCHERS)] - with ExitStack() as stack: - for ctx in contexts: - stack.enter_context(ctx) - with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): - self.add_watcher() + with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): + with ExitStack() as stack: + for _ in range(self.TYPE_MAX_WATCHERS + 1): + stack.enter_context(self.watcher()) class TestCodeObjectWatchers(unittest.TestCase): From 199438b7cc2ef669b8d005d38797477a18b610cb Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 16 Aug 2023 22:17:28 +0300 Subject: [PATCH 160/598] gh-105522: [Enum] Correctly handle possible exceptions during testing (GH-105523) --- Lib/test/test_enum.py | 150 +++++++++++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 54 deletions(-) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a286411f7bcf57..6e128439fa3569 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -38,6 +38,25 @@ def load_tests(loader, tests, ignore): )) return tests +def reraise_if_not_enum(*enum_types_or_exceptions): + from functools import wraps + + def decorator(func): + @wraps(func) + def inner(*args, **kwargs): + excs = [ + e + for e in enum_types_or_exceptions + if isinstance(e, Exception) + ] + if len(excs) == 1: + raise excs[0] + elif excs: + raise ExceptionGroup('Enum Exceptions', excs) + return func(*args, **kwargs) + return inner + return decorator + MODULE = __name__ SHORT_MODULE = MODULE.split('.')[-1] @@ -75,30 +94,42 @@ class FlagStooges(Flag): except Exception as exc: FlagStooges = exc -class FlagStoogesWithZero(Flag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStooges(IntFlag): - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStoogesWithZero(IntFlag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 +try: + class FlagStoogesWithZero(Flag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + FlagStoogesWithZero = exc + +try: + class IntFlagStooges(IntFlag): + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStooges = exc + +try: + class IntFlagStoogesWithZero(IntFlag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStoogesWithZero = exc # for pickle test and subclass tests -class Name(StrEnum): - BDFL = 'Guido van Rossum' - FLUFL = 'Barry Warsaw' +try: + class Name(StrEnum): + BDFL = 'Guido van Rossum' + FLUFL = 'Barry Warsaw' +except Exception as exc: + Name = exc try: Question = Enum('Question', 'who what when where why', module=__name__) @@ -204,26 +235,35 @@ def __get__(self, instance, ownerclass): # for global repr tests -@enum.global_enum -class HeadlightsK(IntFlag, boundary=enum.KEEP): - OFF_K = 0 - LOW_BEAM_K = auto() - HIGH_BEAM_K = auto() - FOG_K = auto() +try: + @enum.global_enum + class HeadlightsK(IntFlag, boundary=enum.KEEP): + OFF_K = 0 + LOW_BEAM_K = auto() + HIGH_BEAM_K = auto() + FOG_K = auto() +except Exception as exc: + HeadlightsK = exc -@enum.global_enum -class HeadlightsC(IntFlag, boundary=enum.CONFORM): - OFF_C = 0 - LOW_BEAM_C = auto() - HIGH_BEAM_C = auto() - FOG_C = auto() +try: + @enum.global_enum + class HeadlightsC(IntFlag, boundary=enum.CONFORM): + OFF_C = 0 + LOW_BEAM_C = auto() + HIGH_BEAM_C = auto() + FOG_C = auto() +except Exception as exc: + HeadlightsC = exc -@enum.global_enum -class NoName(Flag): - ONE = 1 - TWO = 2 +try: + @enum.global_enum + class NoName(Flag): + ONE = 1 + TWO = 2 +except Exception as exc: + NoName = exc # tests @@ -1124,9 +1164,8 @@ def red(self): green = 2 blue = 3 + @reraise_if_not_enum(Theory) def test_enum_function_with_qualname(self): - if isinstance(Theory, Exception): - raise Theory self.assertEqual(Theory.__qualname__, 'spanish_inquisition') def test_enum_of_types(self): @@ -1355,6 +1394,7 @@ class MyUnBrokenEnum(UnBrokenInt, Enum): test_pickle_dump_load(self.assertIs, MyUnBrokenEnum.I) test_pickle_dump_load(self.assertIs, MyUnBrokenEnum) + @reraise_if_not_enum(FloatStooges) def test_floatenum_fromhex(self): h = float.hex(FloatStooges.MOE.value) self.assertIs(FloatStooges.fromhex(h), FloatStooges.MOE) @@ -1475,6 +1515,7 @@ class ThreePart(Enum): self.assertIs(ThreePart((3, 3.0, 'three')), ThreePart.THREE) self.assertIs(ThreePart(3, 3.0, 'three'), ThreePart.THREE) + @reraise_if_not_enum(IntStooges) def test_intenum_from_bytes(self): self.assertIs(IntStooges.from_bytes(b'\x00\x03', 'big'), IntStooges.MOE) with self.assertRaises(ValueError): @@ -1503,33 +1544,28 @@ def repr(self): class Huh(MyStr, MyInt, Enum): One = 1 + @reraise_if_not_enum(Stooges) def test_pickle_enum(self): - if isinstance(Stooges, Exception): - raise Stooges test_pickle_dump_load(self.assertIs, Stooges.CURLY) test_pickle_dump_load(self.assertIs, Stooges) + @reraise_if_not_enum(IntStooges) def test_pickle_int(self): - if isinstance(IntStooges, Exception): - raise IntStooges test_pickle_dump_load(self.assertIs, IntStooges.CURLY) test_pickle_dump_load(self.assertIs, IntStooges) + @reraise_if_not_enum(FloatStooges) def test_pickle_float(self): - if isinstance(FloatStooges, Exception): - raise FloatStooges test_pickle_dump_load(self.assertIs, FloatStooges.CURLY) test_pickle_dump_load(self.assertIs, FloatStooges) + @reraise_if_not_enum(Answer) def test_pickle_enum_function(self): - if isinstance(Answer, Exception): - raise Answer test_pickle_dump_load(self.assertIs, Answer.him) test_pickle_dump_load(self.assertIs, Answer) + @reraise_if_not_enum(Question) def test_pickle_enum_function_with_module(self): - if isinstance(Question, Exception): - raise Question test_pickle_dump_load(self.assertIs, Question.who) test_pickle_dump_load(self.assertIs, Question) @@ -1592,9 +1628,8 @@ class Season(Enum): [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], ) + @reraise_if_not_enum(Name) def test_subclassing(self): - if isinstance(Name, Exception): - raise Name self.assertEqual(Name.BDFL, 'Guido van Rossum') self.assertTrue(Name.BDFL, Name('Guido van Rossum')) self.assertIs(Name.BDFL, getattr(Name, 'BDFL')) @@ -3330,9 +3365,13 @@ def test_programatic_function_from_dict(self): self.assertIn(e, Perm) self.assertIs(type(e), Perm) + @reraise_if_not_enum( + FlagStooges, + FlagStoogesWithZero, + IntFlagStooges, + IntFlagStoogesWithZero, + ) def test_pickle(self): - if isinstance(FlagStooges, Exception): - raise FlagStooges test_pickle_dump_load(self.assertIs, FlagStooges.CURLY) test_pickle_dump_load(self.assertEqual, FlagStooges.CURLY|FlagStooges.MOE) @@ -3637,6 +3676,7 @@ def test_type(self): self.assertTrue(isinstance(Open.WO | Open.RW, Open)) self.assertEqual(Open.WO | Open.RW, 3) + @reraise_if_not_enum(HeadlightsK) def test_global_repr_keep(self): self.assertEqual( repr(HeadlightsK(0)), @@ -3651,6 +3691,7 @@ def test_global_repr_keep(self): '%(m)s.HeadlightsK(8)' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(HeadlightsC) def test_global_repr_conform1(self): self.assertEqual( repr(HeadlightsC(0)), @@ -3665,6 +3706,7 @@ def test_global_repr_conform1(self): '%(m)s.OFF_C' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(NoName) def test_global_enum_str(self): self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') From 57fcf96e4f21b8955b3ae4b4d70e4b756949712f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:23:12 +0100 Subject: [PATCH 161/598] GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (#108047) Remove reference to Distutils in ``cx_Freeze``'s description --- Doc/using/windows.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index d24450e2f963ff..ca79c9d3a9d3a8 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -1246,8 +1246,8 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a ``distutils`` -extension which wraps Python scripts into executable Windows programs +`cx_Freeze `_ +wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install Python. From fb8fe377c4ddaea24ea6aa0a8f5d036986373d39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Aug 2023 22:26:22 +0200 Subject: [PATCH 162/598] gh-107211: Fix select extension build on Solaris (#108012) Export the internal _Py_open() and _Py_write() functions for Solaris: the select shared extension uses them. --- Include/internal/pycore_fileutils.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index cb5ac76772de5e..0ed139f79b1429 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -109,7 +109,8 @@ PyAPI_FUNC(int) _Py_stat( PyObject *path, struct stat *status); -extern int _Py_open( +// Export for 'select' shared extension (Solaris newDevPollObject() uses it) +PyAPI_FUNC(int) _Py_open( const char *pathname, int flags); @@ -126,7 +127,8 @@ extern Py_ssize_t _Py_read( void *buf, size_t count); -extern Py_ssize_t _Py_write( +// Export for 'select' shared extension (Solaris devpoll_flush() uses it) +PyAPI_FUNC(Py_ssize_t) _Py_write( int fd, const void *buf, size_t count); From 882cb79afa2cb11b180ef699fd5cf038e72f6c85 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 16 Aug 2023 23:35:35 +0300 Subject: [PATCH 163/598] gh-56166: Deprecate passing confusing positional arguments in re functions (#107778) Deprecate passing optional arguments maxsplit, count and flags in module-level functions re.split(), re.sub() and re.subn() as positional. They should only be passed by keyword. --- Doc/library/re.rst | 24 ++++--- Doc/whatsnew/3.13.rst | 7 ++ Lib/re/__init__.py | 67 ++++++++++++++++- Lib/test/test_re.py | 71 +++++++++++++++++-- ...3-08-08-16-09-59.gh-issue-56166.WUMhYG.rst | 3 + 5 files changed, 153 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 3f03f0341d8166..ab201e2483f8e3 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -898,7 +898,7 @@ Functions ['Words', 'words', 'words', ''] >>> re.split(r'(\W+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'\W+', 'Words, words, words.', 1) + >>> re.split(r'\W+', 'Words, words, words.', maxsplit=1) ['Words', 'words, words.'] >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) ['0', '3', '9'] @@ -929,6 +929,11 @@ Functions .. versionchanged:: 3.7 Added support of splitting on a pattern that could match an empty string. + .. deprecated:: 3.13 + Passing *maxsplit* and *flags* as positional arguments is deprecated. + In future Python versions they will be + :ref:`keyword-only parameters `. + .. function:: findall(pattern, string, flags=0) @@ -1027,8 +1032,6 @@ Functions .. versionchanged:: 3.7 Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. - - .. versionchanged:: 3.7 Empty matches for the pattern are replaced when adjacent to a previous non-empty match. @@ -1037,18 +1040,17 @@ Functions In :class:`bytes` replacement strings, group *name* can only contain bytes in the ASCII range (``b'\x00'``-``b'\x7f'``). + .. deprecated:: 3.13 + Passing *count* and *flags* as positional arguments is deprecated. + In future Python versions they will be + :ref:`keyword-only parameters `. + .. function:: subn(pattern, repl, string, count=0, flags=0) Perform the same operation as :func:`sub`, but return a tuple ``(new_string, number_of_subs_made)``. - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - .. function:: escape(pattern) @@ -1656,7 +1658,7 @@ because the address has spaces, our splitting pattern, in it: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> [re.split(":? ", entry, 3) for entry in entries] + >>> [re.split(":? ", entry, maxsplit=3) for entry in entries] [['Ross', 'McFluff', '834.345.1254', '155 Elm Street'], ['Ronald', 'Heathmore', '892.345.3428', '436 Finley Avenue'], ['Frank', 'Burger', '925.541.7625', '662 South Dogwood Way'], @@ -1669,7 +1671,7 @@ house number from the street name: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> [re.split(":? ", entry, 4) for entry in entries] + >>> [re.split(":? ", entry, maxsplit=4) for entry in entries] [['Ross', 'McFluff', '834.345.1254', '155', 'Elm Street'], ['Ronald', 'Heathmore', '892.345.3428', '436', 'Finley Avenue'], ['Frank', 'Burger', '925.541.7625', '662', 'South Dogwood Way'], diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 84ffd84b9f0bb6..519dee5eb7d6cf 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -832,6 +832,13 @@ Porting to Python 3.13 Deprecated ---------- +* Passing optional arguments *maxsplit*, *count* and *flags* in module-level + functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional + arguments is now deprecated. + In future Python versions these parameters will be + :ref:`keyword-only `. + (Contributed by Serhiy Storchaka in :gh:`56166`.) + * Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index d6fccd5bc97cc0..428d1b0d5fbd87 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -175,16 +175,39 @@ def search(pattern, string, flags=0): a Match object, or None if no match was found.""" return _compile(pattern, flags).search(string) -def sub(pattern, repl, string, count=0, flags=0): +class _ZeroSentinel(int): + pass +_zero_sentinel = _ZeroSentinel() + +def sub(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel): """Return the string obtained by replacing the leftmost non-overlapping occurrences of the pattern in string by the replacement repl. repl can be either a string or a callable; if a string, backslash escapes in it are processed. If it is a callable, it's passed the Match object and must return a replacement string to be used.""" + if args: + if count is not _zero_sentinel: + raise TypeError("sub() got multiple values for argument 'count'") + count, *args = args + if args: + if flags is not _zero_sentinel: + raise TypeError("sub() got multiple values for argument 'flags'") + flags, *args = args + if args: + raise TypeError("sub() takes from 3 to 5 positional arguments " + "but %d were given" % (5 + len(args))) + + import warnings + warnings.warn( + "'count' is passed as positional argument", + DeprecationWarning, stacklevel=2 + ) + return _compile(pattern, flags).sub(repl, string, count) +sub.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' -def subn(pattern, repl, string, count=0, flags=0): +def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel): """Return a 2-tuple containing (new_string, number). new_string is the string obtained by replacing the leftmost non-overlapping occurrences of the pattern in the source @@ -193,9 +216,28 @@ def subn(pattern, repl, string, count=0, flags=0): callable; if a string, backslash escapes in it are processed. If it is a callable, it's passed the Match object and must return a replacement string to be used.""" + if args: + if count is not _zero_sentinel: + raise TypeError("subn() got multiple values for argument 'count'") + count, *args = args + if args: + if flags is not _zero_sentinel: + raise TypeError("subn() got multiple values for argument 'flags'") + flags, *args = args + if args: + raise TypeError("subn() takes from 3 to 5 positional arguments " + "but %d were given" % (5 + len(args))) + + import warnings + warnings.warn( + "'count' is passed as positional argument", + DeprecationWarning, stacklevel=2 + ) + return _compile(pattern, flags).subn(repl, string, count) +subn.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' -def split(pattern, string, maxsplit=0, flags=0): +def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel): """Split the source string by the occurrences of the pattern, returning a list containing the resulting substrings. If capturing parentheses are used in pattern, then the text of all @@ -203,7 +245,26 @@ def split(pattern, string, maxsplit=0, flags=0): list. If maxsplit is nonzero, at most maxsplit splits occur, and the remainder of the string is returned as the final element of the list.""" + if args: + if maxsplit is not _zero_sentinel: + raise TypeError("split() got multiple values for argument 'maxsplit'") + maxsplit, *args = args + if args: + if flags is not _zero_sentinel: + raise TypeError("split() got multiple values for argument 'flags'") + flags, *args = args + if args: + raise TypeError("split() takes from 2 to 4 positional arguments " + "but %d were given" % (4 + len(args))) + + import warnings + warnings.warn( + "'maxsplit' is passed as positional argument", + DeprecationWarning, stacklevel=2 + ) + return _compile(pattern, flags).split(string, maxsplit) +split.__text_signature__ = '(pattern, string, maxsplit=0, flags=0)' def findall(pattern, string, flags=0): """Return a list of all non-overlapping matches in the string. diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 042f97f57ecf18..45bce1925f9e89 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -127,8 +127,10 @@ def test_basic_re_sub(self): self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x') self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'), '9.3 -3 24x100y') - self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), - '9.3 -3 23x99y') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), + '9.3 -3 23x99y') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', count=3), '9.3 -3 23x99y') @@ -235,9 +237,42 @@ def test_sub_template_numeric_escape(self): def test_qualified_re_sub(self): self.assertEqual(re.sub('a', 'b', 'aaaaa'), 'bbbbb') - self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub('a', 'b', 'aaaaa', count=1), 'baaaa') + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'count'"): + re.sub('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'flags'"): + re.sub('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"sub\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.sub('a', 'b', 'aaaaa', 1, 0, 0) + + def test_misuse_flags(self): + with self.assertWarns(DeprecationWarning) as w: + result = re.sub('a', 'b', 'aaaaa', re.I) + self.assertEqual(result, re.sub('a', 'b', 'aaaaa', count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.subn("b*", "x", "xyz", re.I) + self.assertEqual(result, re.subn("b*", "x", "xyz", count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.split(":", ":a:b::c", re.I) + self.assertEqual(result, re.split(":", ":a:b::c", maxsplit=int(re.I))) + self.assertEqual(str(w.warning), + "'maxsplit' is passed as positional argument") + self.assertEqual(w.filename, __file__) + def test_bug_114660(self): self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'), 'hello there') @@ -344,9 +379,22 @@ def test_re_subn(self): self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) self.assertEqual(re.subn("b+", "x", "xyz"), ('xyz', 0)) self.assertEqual(re.subn("b*", "x", "xyz"), ('xxxyxzx', 4)) - self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + self.assertEqual(w.filename, __file__) self.assertEqual(re.subn("b*", "x", "xyz", count=2), ('xxxyz', 2)) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'count'"): + re.subn('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'flags'"): + re.subn('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"subn\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.subn('a', 'b', 'aaaaa', 1, 0, 0) + def test_re_split(self): for string in ":a:b::c", S(":a:b::c"): self.assertTypedEqual(re.split(":", string), @@ -401,7 +449,9 @@ def test_re_split(self): self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) def test_qualified_re_split(self): - self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + self.assertEqual(w.filename, __file__) self.assertEqual(re.split(":", ":a:b::c", maxsplit=2), ['', 'a', 'b::c']) self.assertEqual(re.split(':', 'a:b:c:d', maxsplit=2), ['a', 'b', 'c:d']) self.assertEqual(re.split("(:)", ":a:b::c", maxsplit=2), @@ -411,6 +461,17 @@ def test_qualified_re_split(self): self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), ['', ':', '', '', 'a:b::c']) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'maxsplit'"): + re.split(":", ":a:b::c", 2, maxsplit=2) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'flags'"): + re.split(":", ":a:b::c", 2, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"split\(\) takes from 2 to 4 positional arguments but 5 " + r"were given"): + re.split(":", ":a:b::c", 2, 0, 0) + def test_re_findall(self): self.assertEqual(re.findall(":+", "abc"), []) for string in "a:b::c:::d", S("a:b::c:::d"): diff --git a/Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst b/Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst new file mode 100644 index 00000000000000..34d776ae8fc3ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst @@ -0,0 +1,3 @@ +Deprecate passing optional arguments *maxsplit*, *count* and *flags* in +module-level functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional. +They should only be passed by keyword. From 902864256cb261428ae9682ca0ffddd597e1f894 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:42:23 +0100 Subject: [PATCH 164/598] GH-92584: Redirect macOS package installation to the PPUG (#108044) --- Doc/using/mac.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 65178272862168..eb1413af2cbc3d 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -125,13 +125,9 @@ http://www.hashcollision.org/hkn/python/idle_intro/index.html. Installing Additional Python Packages ===================================== -There are several methods to install additional Python packages: +This section has moved to the `Python Packaging User Guide`_. -* Packages can be installed via the standard Python distutils mode (``python - setup.py install``). - -* Many packages can also be installed via the :program:`setuptools` extension - or :program:`pip` wrapper, see https://pip.pypa.io/. +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ GUI Programming on the Mac From 636ca313b2f7ce09a311889995778dccae8ebe40 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:43:30 +0100 Subject: [PATCH 165/598] GH-92584: Remove references to Distutils in ``PYTHONUSERBASE`` (#108040) Remove references to Distutils in ``PYTHONUSERBASE`` --- Doc/using/cmdline.rst | 4 ++-- Misc/python.man | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4bf67eb439ec6c..23c89400f152b1 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,8 +811,8 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`Distutils installation paths ` for - ``python setup.py install --user``. + and :ref:`installation paths ` for + ``python -m pip install --user``. .. seealso:: diff --git a/Misc/python.man b/Misc/python.man index bf7cf767d164a6..9f89c94adf5028 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -592,8 +592,8 @@ works on Mac OS X. .IP PYTHONUSERBASE Defines the user base directory, which is used to compute the path of the user .IR site-packages -directory and Distutils installation paths for -.IR "python setup\.py install \-\-user" . +directory and installation paths for +.IR "python \-m pip install \-\-user" . .IP PYTHONPROFILEIMPORTTIME If this environment variable is set to a non-empty string, Python will show how long each import takes. This is exactly equivalent to setting From fbb7cbc0e92168077fd56de942901511e99ca60a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:06:56 +0100 Subject: [PATCH 166/598] GH-92584: Remove Installing Python Modules (Distutils version) (#108020) --- Doc/contents.rst | 7 - Doc/extending/building.rst | 1 + Doc/install/index.rst | 1081 ------------------------------------ Doc/library/site.rst | 2 +- Doc/using/cmdline.rst | 2 +- 5 files changed, 3 insertions(+), 1090 deletions(-) delete mode 100644 Doc/install/index.rst diff --git a/Doc/contents.rst b/Doc/contents.rst index 464f93bdf85f95..649a1344a0b26a 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -21,10 +21,3 @@ bugs.rst copyright.rst license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - install/index.rst diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 880bb33ee56718..ddde567f6f3efa 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -45,6 +45,7 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. .. highlight:: c +.. _install-index: .. _setuptools-index: Building C and C++ Extensions with setuptools diff --git a/Doc/install/index.rst b/Doc/install/index.rst deleted file mode 100644 index ffb4a202fe89f2..00000000000000 --- a/Doc/install/index.rst +++ /dev/null @@ -1,1081 +0,0 @@ -.. highlight:: none - -.. _install-index: - -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward - -.. TODO: Fill in XXX comments - -.. note:: - - The entire ``distutils`` package has been deprecated and will be - removed in Python 3.12. This documentation is retained as a - reference only, and will be removed with the package. See the - :ref:`What's New ` entry for more information. - -.. seealso:: - - :ref:`installing-index` - The up to date module installation documentation. For regular Python - usage, you almost certainly want that document rather than this one. - -.. note:: - - This document is being retained solely until the ``setuptools`` documentation - at https://setuptools.readthedocs.io/en/latest/setuptools.html - independently covers all of the relevant information currently included here. - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. - - -.. _inst-intro: - - -Introduction -============ - -In Python 2.0, the ``distutils`` API was first added to the standard library. -This provided Linux distro maintainers with a standard way of converting -Python projects into Linux distro packages, and system administrators with a -standard way of installing them directly onto target systems. - -In the many years since Python 2.0 was released, tightly coupling the build -system and package installer to the language runtime release cycle has turned -out to be problematic, and it is now recommended that projects use the -``pip`` package installer and the ``setuptools`` build system, rather than -using ``distutils`` directly. - -See :ref:`installing-index` and :ref:`distributing-index` for more details. - -This legacy documentation is being retained only until we're confident that the -``setuptools`` documentation covers everything needed. - -.. _inst-new-standard: - -Distutils based source distributions ------------------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`!--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and macOS, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and macOS. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`: - -.. code-block:: pycon - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :const:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :const:`site.USER_SITE`). -Here are the values for UNIX, including macOS: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`!--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`!sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`!--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`!--prefix` option defines the installation base, and -the :option:`!--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to -:option:`!--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`!--prefix` -and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`!--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`!--prefix` option; the -:option:`!--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the -same---they only differ when you use the Unix "prefix scheme" and supply -different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` -will override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`!--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`!--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme", where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation -directory---you just have to remember that there are two types of modules -to worry about, Python and extension modules, which can conveniently be both -controlled by one option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`): - -.. code-block:: ini - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, - -.. code-block:: ini - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it: - -.. code-block:: python - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same minor version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and macOS, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the ``--no-user-cfg`` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`~pwd.getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default: - -.. code-block:: ini - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following: - -.. code-block:: ini - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`!--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`!--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or - :option:`!-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`!-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`!-o32` option, and the linker will be passed -:option:`!-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check https://www.sourceware.org/cygwin/ for more information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index ff9e9107f9d163..ea3b2e996574ef 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -191,7 +191,7 @@ Module contents :file:`~/Library/Python/{X.Y}` for macOS framework builds, and :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. + etc. for the user installation scheme. See also :envvar:`PYTHONUSERBASE`. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 23c89400f152b1..921b6a6961c7b2 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,7 +811,7 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`installation paths ` for + and installation paths for ``python -m pip install --user``. .. seealso:: From e88eb3775ecdcb3af6c6d694a935b7fa5f41e5ce Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:13:05 +0100 Subject: [PATCH 167/598] GH-92584: Remove references to Distutils in configure.rst (#108043) Remove references to Distutils in configure.rst --- Doc/using/configure.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 924e73dc54da2e..441d346a1a38a0 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -686,7 +686,6 @@ Main files of the build system * :file:`pyconfig.h` (created by :file:`configure`); * :file:`Modules/Setup`: C extensions built by the Makefile using :file:`Module/makesetup` shell script; -* :file:`setup.py`: C extensions built using the ``setuptools`` package. Main build steps ---------------- @@ -695,8 +694,7 @@ Main build steps * A static ``libpython`` library (``.a``) is created from objects files. * ``python.o`` and the static ``libpython`` library are linked into the final ``python`` program. -* C extensions are built by the Makefile (see :file:`Modules/Setup`) - and ``python setup.py build``. +* C extensions are built by the Makefile (see :file:`Modules/Setup`). Main Makefile targets --------------------- @@ -748,9 +746,6 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :file:`setup.py` script only builds C extensions as shared libraries using -the :mod:`distutils` module. - The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and :c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: @@ -784,7 +779,7 @@ Preprocessor flags headers in a nonstandard directory ````. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: BASECPPFLAGS @@ -821,8 +816,8 @@ Compiler flags .. envvar:: CFLAGS_NODIST :envvar:`CFLAGS_NODIST` is used for building the interpreter and stdlib C - extensions. Use it when a compiler flag should *not* be part of the - distutils :envvar:`CFLAGS` once Python is installed (:issue:`21121`). + extensions. Use it when a compiler flag should *not* be part of + :envvar:`CFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`CFLAGS` should not contain: @@ -952,7 +947,7 @@ Linker flags :envvar:`LDFLAGS_NODIST` is used in the same manner as :envvar:`CFLAGS_NODIST`. Use it when a linker flag should *not* be part of - the distutils :envvar:`LDFLAGS` once Python is installed (:issue:`35257`). + :envvar:`LDFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`LDFLAGS` should not contain: @@ -974,7 +969,7 @@ Linker flags directory ````. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: LIBS From 665a4391e10167dad1c854fb604c86f336fcd331 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 16 Aug 2023 23:25:18 +0100 Subject: [PATCH 168/598] gh-105481: generate op IDs from bytecode.c instead of hard coding them in opcode.py (#107971) --- Include/internal/pycore_opcode.h | 540 ------------------ Include/internal/pycore_opcode_metadata.h | 490 ++++++++++++++++ Include/opcode_ids.h | 436 +++++++------- Lib/_opcode_metadata.py | 226 +++++++- Lib/dis.py | 10 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 218 +------ Lib/test/test_dis.py | 369 ++++++------ Lib/test/test_embed.py | 2 +- Makefile.pre.in | 9 +- ...-08-15-13-06-05.gh-issue-107971.lPbx04.rst | 2 + Objects/codeobject.c | 3 +- Objects/frameobject.c | 2 + PCbuild/regen.targets | 4 +- Programs/test_frozenmain.h | 22 +- Python/bytecodes.c | 1 - Python/opcode_targets.h | 333 ++++++----- Tools/build/deepfreeze.py | 8 +- Tools/build/generate_opcode_h.py | 106 +--- Tools/cases_generator/generate_cases.py | 206 ++++++- Tools/scripts/summarize_stats.py | 4 +- 21 files changed, 1533 insertions(+), 1461 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index a187da6e24730f..b47e796485236c 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -14,8 +14,6 @@ extern "C" { extern const uint8_t _PyOpcode_Caches[256]; -extern const uint8_t _PyOpcode_Deopt[256]; - #ifdef NEED_OPCODE_TABLES const uint8_t _PyOpcode_Caches[256] = { @@ -34,546 +32,8 @@ const uint8_t _PyOpcode_Caches[256] = { [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, }; - -const uint8_t _PyOpcode_Deopt[256] = { - [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, - [BEFORE_WITH] = BEFORE_WITH, - [BINARY_OP] = BINARY_OP, - [BINARY_OP_ADD_FLOAT] = BINARY_OP, - [BINARY_OP_ADD_INT] = BINARY_OP, - [BINARY_OP_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, - [BINARY_OP_MULTIPLY_INT] = BINARY_OP, - [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, - [BINARY_OP_SUBTRACT_INT] = BINARY_OP, - [BINARY_SLICE] = BINARY_SLICE, - [BINARY_SUBSCR] = BINARY_SUBSCR, - [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, - [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, - [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, - [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, - [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, - [BUILD_LIST] = BUILD_LIST, - [BUILD_MAP] = BUILD_MAP, - [BUILD_SET] = BUILD_SET, - [BUILD_SLICE] = BUILD_SLICE, - [BUILD_STRING] = BUILD_STRING, - [BUILD_TUPLE] = BUILD_TUPLE, - [CACHE] = CACHE, - [CALL] = CALL, - [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, - [CALL_BUILTIN_CLASS] = CALL, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, - [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, - [CALL_NO_KW_BUILTIN_FAST] = CALL, - [CALL_NO_KW_BUILTIN_O] = CALL, - [CALL_NO_KW_ISINSTANCE] = CALL, - [CALL_NO_KW_LEN] = CALL, - [CALL_NO_KW_LIST_APPEND] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = CALL, - [CALL_NO_KW_STR_1] = CALL, - [CALL_NO_KW_TUPLE_1] = CALL, - [CALL_NO_KW_TYPE_1] = CALL, - [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, - [CHECK_EG_MATCH] = CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, - [CLEANUP_THROW] = CLEANUP_THROW, - [COMPARE_OP] = COMPARE_OP, - [COMPARE_OP_FLOAT] = COMPARE_OP, - [COMPARE_OP_INT] = COMPARE_OP, - [COMPARE_OP_STR] = COMPARE_OP, - [CONTAINS_OP] = CONTAINS_OP, - [CONVERT_VALUE] = CONVERT_VALUE, - [COPY] = COPY, - [COPY_FREE_VARS] = COPY_FREE_VARS, - [DELETE_ATTR] = DELETE_ATTR, - [DELETE_DEREF] = DELETE_DEREF, - [DELETE_FAST] = DELETE_FAST, - [DELETE_GLOBAL] = DELETE_GLOBAL, - [DELETE_NAME] = DELETE_NAME, - [DELETE_SUBSCR] = DELETE_SUBSCR, - [DICT_MERGE] = DICT_MERGE, - [DICT_UPDATE] = DICT_UPDATE, - [END_ASYNC_FOR] = END_ASYNC_FOR, - [END_FOR] = END_FOR, - [END_SEND] = END_SEND, - [ENTER_EXECUTOR] = ENTER_EXECUTOR, - [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, - [EXTENDED_ARG] = EXTENDED_ARG, - [FORMAT_SIMPLE] = FORMAT_SIMPLE, - [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, - [FOR_ITER] = FOR_ITER, - [FOR_ITER_GEN] = FOR_ITER, - [FOR_ITER_LIST] = FOR_ITER, - [FOR_ITER_RANGE] = FOR_ITER, - [FOR_ITER_TUPLE] = FOR_ITER, - [GET_AITER] = GET_AITER, - [GET_ANEXT] = GET_ANEXT, - [GET_AWAITABLE] = GET_AWAITABLE, - [GET_ITER] = GET_ITER, - [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, - [IMPORT_FROM] = IMPORT_FROM, - [IMPORT_NAME] = IMPORT_NAME, - [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, - [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, - [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = INTERPRETER_EXIT, - [IS_OP] = IS_OP, - [JUMP_BACKWARD] = JUMP_BACKWARD, - [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_FORWARD] = JUMP_FORWARD, - [KW_NAMES] = KW_NAMES, - [LIST_APPEND] = LIST_APPEND, - [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, - [LOAD_ATTR] = LOAD_ATTR, - [LOAD_ATTR_CLASS] = LOAD_ATTR, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, - [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, - [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, - [LOAD_ATTR_MODULE] = LOAD_ATTR, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = LOAD_ATTR, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = LOAD_ATTR, - [LOAD_ATTR_PROPERTY] = LOAD_ATTR, - [LOAD_ATTR_SLOT] = LOAD_ATTR, - [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, - [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, - [LOAD_CONST] = LOAD_CONST, - [LOAD_DEREF] = LOAD_DEREF, - [LOAD_FAST] = LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, - [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, - [LOAD_FAST_LOAD_FAST] = LOAD_FAST_LOAD_FAST, - [LOAD_FROM_DICT_OR_DEREF] = LOAD_FROM_DICT_OR_DEREF, - [LOAD_FROM_DICT_OR_GLOBALS] = LOAD_FROM_DICT_OR_GLOBALS, - [LOAD_GLOBAL] = LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, - [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, - [LOAD_LOCALS] = LOAD_LOCALS, - [LOAD_NAME] = LOAD_NAME, - [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, - [MAKE_CELL] = MAKE_CELL, - [MAKE_FUNCTION] = MAKE_FUNCTION, - [MAP_ADD] = MAP_ADD, - [MATCH_CLASS] = MATCH_CLASS, - [MATCH_KEYS] = MATCH_KEYS, - [MATCH_MAPPING] = MATCH_MAPPING, - [MATCH_SEQUENCE] = MATCH_SEQUENCE, - [NOP] = NOP, - [POP_EXCEPT] = POP_EXCEPT, - [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, - [POP_TOP] = POP_TOP, - [PUSH_EXC_INFO] = PUSH_EXC_INFO, - [PUSH_NULL] = PUSH_NULL, - [RAISE_VARARGS] = RAISE_VARARGS, - [RERAISE] = RERAISE, - [RESERVED] = RESERVED, - [RESUME] = RESUME, - [RETURN_CONST] = RETURN_CONST, - [RETURN_GENERATOR] = RETURN_GENERATOR, - [RETURN_VALUE] = RETURN_VALUE, - [SEND] = SEND, - [SEND_GEN] = SEND, - [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, - [SET_ADD] = SET_ADD, - [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, - [SET_UPDATE] = SET_UPDATE, - [STORE_ATTR] = STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, - [STORE_ATTR_SLOT] = STORE_ATTR, - [STORE_ATTR_WITH_HINT] = STORE_ATTR, - [STORE_DEREF] = STORE_DEREF, - [STORE_FAST] = STORE_FAST, - [STORE_FAST_LOAD_FAST] = STORE_FAST_LOAD_FAST, - [STORE_FAST_STORE_FAST] = STORE_FAST_STORE_FAST, - [STORE_GLOBAL] = STORE_GLOBAL, - [STORE_NAME] = STORE_NAME, - [STORE_SLICE] = STORE_SLICE, - [STORE_SUBSCR] = STORE_SUBSCR, - [STORE_SUBSCR_DICT] = STORE_SUBSCR, - [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, - [SWAP] = SWAP, - [TO_BOOL] = TO_BOOL, - [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, - [TO_BOOL_BOOL] = TO_BOOL, - [TO_BOOL_INT] = TO_BOOL, - [TO_BOOL_LIST] = TO_BOOL, - [TO_BOOL_NONE] = TO_BOOL, - [TO_BOOL_STR] = TO_BOOL, - [UNARY_INVERT] = UNARY_INVERT, - [UNARY_NEGATIVE] = UNARY_NEGATIVE, - [UNARY_NOT] = UNARY_NOT, - [UNPACK_EX] = UNPACK_EX, - [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, - [WITH_EXCEPT_START] = WITH_EXCEPT_START, - [YIELD_VALUE] = YIELD_VALUE, -}; -#endif // NEED_OPCODE_TABLES - - -extern const char *const _PyOpcode_OpName[268]; - -#ifdef NEED_OPCODE_TABLES -const char *const _PyOpcode_OpName[268] = { - [CACHE] = "CACHE", - [POP_TOP] = "POP_TOP", - [PUSH_NULL] = "PUSH_NULL", - [INTERPRETER_EXIT] = "INTERPRETER_EXIT", - [END_FOR] = "END_FOR", - [END_SEND] = "END_SEND", - [TO_BOOL] = "TO_BOOL", - [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", - [TO_BOOL_BOOL] = "TO_BOOL_BOOL", - [NOP] = "NOP", - [TO_BOOL_INT] = "TO_BOOL_INT", - [UNARY_NEGATIVE] = "UNARY_NEGATIVE", - [UNARY_NOT] = "UNARY_NOT", - [TO_BOOL_LIST] = "TO_BOOL_LIST", - [TO_BOOL_NONE] = "TO_BOOL_NONE", - [UNARY_INVERT] = "UNARY_INVERT", - [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", - [RESERVED] = "RESERVED", - [TO_BOOL_STR] = "TO_BOOL_STR", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", - [MAKE_FUNCTION] = "MAKE_FUNCTION", - [BINARY_SUBSCR] = "BINARY_SUBSCR", - [BINARY_SLICE] = "BINARY_SLICE", - [STORE_SLICE] = "STORE_SLICE", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [GET_LEN] = "GET_LEN", - [MATCH_MAPPING] = "MATCH_MAPPING", - [MATCH_SEQUENCE] = "MATCH_SEQUENCE", - [MATCH_KEYS] = "MATCH_KEYS", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [PUSH_EXC_INFO] = "PUSH_EXC_INFO", - [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", - [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", - [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", - [FORMAT_SIMPLE] = "FORMAT_SIMPLE", - [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", - [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", - [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", - [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [SEND_GEN] = "SEND_GEN", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [WITH_EXCEPT_START] = "WITH_EXCEPT_START", - [GET_AITER] = "GET_AITER", - [GET_ANEXT] = "GET_ANEXT", - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", - [END_ASYNC_FOR] = "END_ASYNC_FOR", - [CLEANUP_THROW] = "CLEANUP_THROW", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", - [LOAD_LOCALS] = "LOAD_LOCALS", - [COMPARE_OP_INT] = "COMPARE_OP_INT", - [POP_EXCEPT] = "POP_EXCEPT", - [STORE_NAME] = "STORE_NAME", - [DELETE_NAME] = "DELETE_NAME", - [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", - [FOR_ITER] = "FOR_ITER", - [UNPACK_EX] = "UNPACK_EX", - [STORE_ATTR] = "STORE_ATTR", - [DELETE_ATTR] = "DELETE_ATTR", - [STORE_GLOBAL] = "STORE_GLOBAL", - [DELETE_GLOBAL] = "DELETE_GLOBAL", - [SWAP] = "SWAP", - [LOAD_CONST] = "LOAD_CONST", - [LOAD_NAME] = "LOAD_NAME", - [BUILD_TUPLE] = "BUILD_TUPLE", - [BUILD_LIST] = "BUILD_LIST", - [BUILD_SET] = "BUILD_SET", - [BUILD_MAP] = "BUILD_MAP", - [LOAD_ATTR] = "LOAD_ATTR", - [COMPARE_OP] = "COMPARE_OP", - [IMPORT_NAME] = "IMPORT_NAME", - [IMPORT_FROM] = "IMPORT_FROM", - [JUMP_FORWARD] = "JUMP_FORWARD", - [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", - [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", - [LOAD_GLOBAL] = "LOAD_GLOBAL", - [IS_OP] = "IS_OP", - [CONTAINS_OP] = "CONTAINS_OP", - [RERAISE] = "RERAISE", - [COPY] = "COPY", - [RETURN_CONST] = "RETURN_CONST", - [BINARY_OP] = "BINARY_OP", - [SEND] = "SEND", - [LOAD_FAST] = "LOAD_FAST", - [STORE_FAST] = "STORE_FAST", - [DELETE_FAST] = "DELETE_FAST", - [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", - [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", - [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", - [RAISE_VARARGS] = "RAISE_VARARGS", - [GET_AWAITABLE] = "GET_AWAITABLE", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [BUILD_SLICE] = "BUILD_SLICE", - [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", - [MAKE_CELL] = "MAKE_CELL", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [LOAD_DEREF] = "LOAD_DEREF", - [STORE_DEREF] = "STORE_DEREF", - [DELETE_DEREF] = "DELETE_DEREF", - [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", - [EXTENDED_ARG] = "EXTENDED_ARG", - [LIST_APPEND] = "LIST_APPEND", - [SET_ADD] = "SET_ADD", - [MAP_ADD] = "MAP_ADD", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [COPY_FREE_VARS] = "COPY_FREE_VARS", - [YIELD_VALUE] = "YIELD_VALUE", - [RESUME] = "RESUME", - [MATCH_CLASS] = "MATCH_CLASS", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", - [BUILD_STRING] = "BUILD_STRING", - [CONVERT_VALUE] = "CONVERT_VALUE", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [LIST_EXTEND] = "LIST_EXTEND", - [SET_UPDATE] = "SET_UPDATE", - [DICT_MERGE] = "DICT_MERGE", - [DICT_UPDATE] = "DICT_UPDATE", - [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", - [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", - [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", - [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", - [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", - [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", - [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", - [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", - [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", - [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", - [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", - [187] = "<187>", - [188] = "<188>", - [189] = "<189>", - [190] = "<190>", - [191] = "<191>", - [192] = "<192>", - [193] = "<193>", - [194] = "<194>", - [195] = "<195>", - [196] = "<196>", - [197] = "<197>", - [198] = "<198>", - [199] = "<199>", - [200] = "<200>", - [201] = "<201>", - [202] = "<202>", - [203] = "<203>", - [204] = "<204>", - [205] = "<205>", - [206] = "<206>", - [207] = "<207>", - [208] = "<208>", - [209] = "<209>", - [210] = "<210>", - [211] = "<211>", - [212] = "<212>", - [213] = "<213>", - [214] = "<214>", - [215] = "<215>", - [216] = "<216>", - [217] = "<217>", - [218] = "<218>", - [219] = "<219>", - [220] = "<220>", - [221] = "<221>", - [222] = "<222>", - [223] = "<223>", - [224] = "<224>", - [225] = "<225>", - [226] = "<226>", - [227] = "<227>", - [228] = "<228>", - [229] = "<229>", - [ENTER_EXECUTOR] = "ENTER_EXECUTOR", - [231] = "<231>", - [232] = "<232>", - [233] = "<233>", - [234] = "<234>", - [235] = "<235>", - [236] = "<236>", - [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", - [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", - [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", - [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", - [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", - [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", - [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", - [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", - [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [255] = "<255>", - [SETUP_FINALLY] = "SETUP_FINALLY", - [SETUP_CLEANUP] = "SETUP_CLEANUP", - [SETUP_WITH] = "SETUP_WITH", - [POP_BLOCK] = "POP_BLOCK", - [JUMP] = "JUMP", - [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", - [LOAD_METHOD] = "LOAD_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", - [LOAD_CLOSURE] = "LOAD_CLOSURE", -}; #endif // NEED_OPCODE_TABLES -#define EXTRA_CASES \ - case 187: \ - case 188: \ - case 189: \ - case 190: \ - case 191: \ - case 192: \ - case 193: \ - case 194: \ - case 195: \ - case 196: \ - case 197: \ - case 198: \ - case 199: \ - case 200: \ - case 201: \ - case 202: \ - case 203: \ - case 204: \ - case 205: \ - case 206: \ - case 207: \ - case 208: \ - case 209: \ - case 210: \ - case 211: \ - case 212: \ - case 213: \ - case 214: \ - case 215: \ - case 216: \ - case 217: \ - case 218: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ - case 225: \ - case 226: \ - case 227: \ - case 228: \ - case 229: \ - case 231: \ - case 232: \ - case 233: \ - case 234: \ - case 235: \ - case 236: \ - case 255: \ - ; - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index df9b4184e2a50e..7dbd78b32a8957 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1537,3 +1537,493 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [INSERT] = "INSERT", }; #endif // NEED_OPCODE_METADATA + +extern const char *const _PyOpcode_OpName[268]; +#ifdef NEED_OPCODE_METADATA +const char *const _PyOpcode_OpName[268] = { + [CACHE] = "CACHE", + [RESERVED] = "RESERVED", + [RESUME] = "RESUME", + [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", + [BEFORE_WITH] = "BEFORE_WITH", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SLICE] = "BINARY_SLICE", + [BINARY_SUBSCR] = "BINARY_SUBSCR", + [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", + [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", + [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", + [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", + [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", + [CLEANUP_THROW] = "CLEANUP_THROW", + [DELETE_SUBSCR] = "DELETE_SUBSCR", + [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", + [FORMAT_SIMPLE] = "FORMAT_SIMPLE", + [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", + [GET_AITER] = "GET_AITER", + [GET_ANEXT] = "GET_ANEXT", + [GET_ITER] = "GET_ITER", + [GET_LEN] = "GET_LEN", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_LOCALS] = "LOAD_LOCALS", + [MAKE_FUNCTION] = "MAKE_FUNCTION", + [MATCH_KEYS] = "MATCH_KEYS", + [MATCH_MAPPING] = "MATCH_MAPPING", + [MATCH_SEQUENCE] = "MATCH_SEQUENCE", + [NOP] = "NOP", + [POP_EXCEPT] = "POP_EXCEPT", + [POP_TOP] = "POP_TOP", + [PUSH_EXC_INFO] = "PUSH_EXC_INFO", + [PUSH_NULL] = "PUSH_NULL", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [RETURN_VALUE] = "RETURN_VALUE", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR] = "STORE_SUBSCR", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [TO_BOOL] = "TO_BOOL", + [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", + [TO_BOOL_BOOL] = "TO_BOOL_BOOL", + [TO_BOOL_INT] = "TO_BOOL_INT", + [TO_BOOL_LIST] = "TO_BOOL_LIST", + [TO_BOOL_NONE] = "TO_BOOL_NONE", + [TO_BOOL_STR] = "TO_BOOL_STR", + [UNARY_INVERT] = "UNARY_INVERT", + [UNARY_NEGATIVE] = "UNARY_NEGATIVE", + [UNARY_NOT] = "UNARY_NOT", + [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [BINARY_OP] = "BINARY_OP", + [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", + [BUILD_LIST] = "BUILD_LIST", + [BUILD_MAP] = "BUILD_MAP", + [BUILD_SET] = "BUILD_SET", + [BUILD_SLICE] = "BUILD_SLICE", + [BUILD_STRING] = "BUILD_STRING", + [BUILD_TUPLE] = "BUILD_TUPLE", + [CALL] = "CALL", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", + [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", + [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", + [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", + [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [COMPARE_OP] = "COMPARE_OP", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_STR] = "COMPARE_OP_STR", + [CONTAINS_OP] = "CONTAINS_OP", + [CONVERT_VALUE] = "CONVERT_VALUE", + [COPY] = "COPY", + [COPY_FREE_VARS] = "COPY_FREE_VARS", + [DELETE_ATTR] = "DELETE_ATTR", + [DELETE_DEREF] = "DELETE_DEREF", + [DELETE_FAST] = "DELETE_FAST", + [DELETE_GLOBAL] = "DELETE_GLOBAL", + [DELETE_NAME] = "DELETE_NAME", + [DICT_MERGE] = "DICT_MERGE", + [DICT_UPDATE] = "DICT_UPDATE", + [ENTER_EXECUTOR] = "ENTER_EXECUTOR", + [EXTENDED_ARG] = "EXTENDED_ARG", + [FOR_ITER] = "FOR_ITER", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [GET_AWAITABLE] = "GET_AWAITABLE", + [IMPORT_FROM] = "IMPORT_FROM", + [IMPORT_NAME] = "IMPORT_NAME", + [IS_OP] = "IS_OP", + [JUMP_BACKWARD] = "JUMP_BACKWARD", + [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", + [JUMP_FORWARD] = "JUMP_FORWARD", + [KW_NAMES] = "KW_NAMES", + [LIST_APPEND] = "LIST_APPEND", + [LIST_EXTEND] = "LIST_EXTEND", + [LOAD_ATTR] = "LOAD_ATTR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_CONST] = "LOAD_CONST", + [LOAD_DEREF] = "LOAD_DEREF", + [LOAD_FAST] = "LOAD_FAST", + [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", + [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", + [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", + [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", + [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", + [LOAD_GLOBAL] = "LOAD_GLOBAL", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_NAME] = "LOAD_NAME", + [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", + [MAKE_CELL] = "MAKE_CELL", + [MAP_ADD] = "MAP_ADD", + [MATCH_CLASS] = "MATCH_CLASS", + [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", + [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", + [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", + [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [RAISE_VARARGS] = "RAISE_VARARGS", + [RERAISE] = "RERAISE", + [RETURN_CONST] = "RETURN_CONST", + [SEND] = "SEND", + [SEND_GEN] = "SEND_GEN", + [SET_ADD] = "SET_ADD", + [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", + [SET_UPDATE] = "SET_UPDATE", + [STORE_ATTR] = "STORE_ATTR", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_DEREF] = "STORE_DEREF", + [STORE_FAST] = "STORE_FAST", + [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", + [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", + [STORE_GLOBAL] = "STORE_GLOBAL", + [STORE_NAME] = "STORE_NAME", + [SWAP] = "SWAP", + [UNPACK_EX] = "UNPACK_EX", + [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [YIELD_VALUE] = "YIELD_VALUE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [JUMP] = "JUMP", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", + [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_METHOD] = "LOAD_METHOD", + [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", + [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", + [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", + [POP_BLOCK] = "POP_BLOCK", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_WITH] = "SETUP_WITH", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", +}; +#endif // NEED_OPCODE_METADATA + +extern const uint8_t _PyOpcode_Deopt[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Deopt[256] = { + [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, + [BEFORE_WITH] = BEFORE_WITH, + [BINARY_OP] = BINARY_OP, + [BINARY_OP_ADD_FLOAT] = BINARY_OP, + [BINARY_OP_ADD_INT] = BINARY_OP, + [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, + [BINARY_OP_MULTIPLY_INT] = BINARY_OP, + [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, + [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_SLICE] = BINARY_SLICE, + [BINARY_SUBSCR] = BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, + [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, + [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, + [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, + [BUILD_LIST] = BUILD_LIST, + [BUILD_MAP] = BUILD_MAP, + [BUILD_SET] = BUILD_SET, + [BUILD_SLICE] = BUILD_SLICE, + [BUILD_STRING] = BUILD_STRING, + [BUILD_TUPLE] = BUILD_TUPLE, + [CACHE] = CACHE, + [CALL] = CALL, + [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BUILTIN_CLASS] = CALL, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, + [CALL_NO_KW_BUILTIN_FAST] = CALL, + [CALL_NO_KW_BUILTIN_O] = CALL, + [CALL_NO_KW_ISINSTANCE] = CALL, + [CALL_NO_KW_LEN] = CALL, + [CALL_NO_KW_LIST_APPEND] = CALL, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = CALL, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = CALL, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NO_KW_STR_1] = CALL, + [CALL_NO_KW_TUPLE_1] = CALL, + [CALL_NO_KW_TYPE_1] = CALL, + [CALL_PY_EXACT_ARGS] = CALL, + [CALL_PY_WITH_DEFAULTS] = CALL, + [CHECK_EG_MATCH] = CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, + [CLEANUP_THROW] = CLEANUP_THROW, + [COMPARE_OP] = COMPARE_OP, + [COMPARE_OP_FLOAT] = COMPARE_OP, + [COMPARE_OP_INT] = COMPARE_OP, + [COMPARE_OP_STR] = COMPARE_OP, + [CONTAINS_OP] = CONTAINS_OP, + [CONVERT_VALUE] = CONVERT_VALUE, + [COPY] = COPY, + [COPY_FREE_VARS] = COPY_FREE_VARS, + [DELETE_ATTR] = DELETE_ATTR, + [DELETE_DEREF] = DELETE_DEREF, + [DELETE_FAST] = DELETE_FAST, + [DELETE_GLOBAL] = DELETE_GLOBAL, + [DELETE_NAME] = DELETE_NAME, + [DELETE_SUBSCR] = DELETE_SUBSCR, + [DICT_MERGE] = DICT_MERGE, + [DICT_UPDATE] = DICT_UPDATE, + [END_ASYNC_FOR] = END_ASYNC_FOR, + [END_FOR] = END_FOR, + [END_SEND] = END_SEND, + [ENTER_EXECUTOR] = ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, + [EXTENDED_ARG] = EXTENDED_ARG, + [FORMAT_SIMPLE] = FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, + [FOR_ITER] = FOR_ITER, + [FOR_ITER_GEN] = FOR_ITER, + [FOR_ITER_LIST] = FOR_ITER, + [FOR_ITER_RANGE] = FOR_ITER, + [FOR_ITER_TUPLE] = FOR_ITER, + [GET_AITER] = GET_AITER, + [GET_ANEXT] = GET_ANEXT, + [GET_AWAITABLE] = GET_AWAITABLE, + [GET_ITER] = GET_ITER, + [GET_LEN] = GET_LEN, + [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, + [IMPORT_FROM] = IMPORT_FROM, + [IMPORT_NAME] = IMPORT_NAME, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = INTERPRETER_EXIT, + [IS_OP] = IS_OP, + [JUMP_BACKWARD] = JUMP_BACKWARD, + [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_FORWARD] = JUMP_FORWARD, + [KW_NAMES] = KW_NAMES, + [LIST_APPEND] = LIST_APPEND, + [LIST_EXTEND] = LIST_EXTEND, + [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, + [LOAD_ATTR] = LOAD_ATTR, + [LOAD_ATTR_CLASS] = LOAD_ATTR, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, + [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, + [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, + [LOAD_ATTR_MODULE] = LOAD_ATTR, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = LOAD_ATTR, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = LOAD_ATTR, + [LOAD_ATTR_PROPERTY] = LOAD_ATTR, + [LOAD_ATTR_SLOT] = LOAD_ATTR, + [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, + [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_CONST] = LOAD_CONST, + [LOAD_DEREF] = LOAD_DEREF, + [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, + [LOAD_LOCALS] = LOAD_LOCALS, + [LOAD_NAME] = LOAD_NAME, + [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, + [MAKE_CELL] = MAKE_CELL, + [MAKE_FUNCTION] = MAKE_FUNCTION, + [MAP_ADD] = MAP_ADD, + [MATCH_CLASS] = MATCH_CLASS, + [MATCH_KEYS] = MATCH_KEYS, + [MATCH_MAPPING] = MATCH_MAPPING, + [MATCH_SEQUENCE] = MATCH_SEQUENCE, + [NOP] = NOP, + [POP_EXCEPT] = POP_EXCEPT, + [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, + [POP_TOP] = POP_TOP, + [PUSH_EXC_INFO] = PUSH_EXC_INFO, + [PUSH_NULL] = PUSH_NULL, + [RAISE_VARARGS] = RAISE_VARARGS, + [RERAISE] = RERAISE, + [RESERVED] = RESERVED, + [RESUME] = RESUME, + [RETURN_CONST] = RETURN_CONST, + [RETURN_GENERATOR] = RETURN_GENERATOR, + [RETURN_VALUE] = RETURN_VALUE, + [SEND] = SEND, + [SEND_GEN] = SEND, + [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, + [SET_ADD] = SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = SET_UPDATE, + [STORE_ATTR] = STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, + [STORE_ATTR_SLOT] = STORE_ATTR, + [STORE_ATTR_WITH_HINT] = STORE_ATTR, + [STORE_DEREF] = STORE_DEREF, + [STORE_FAST] = STORE_FAST, + [STORE_FAST_LOAD_FAST] = STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = STORE_GLOBAL, + [STORE_NAME] = STORE_NAME, + [STORE_SLICE] = STORE_SLICE, + [STORE_SUBSCR] = STORE_SUBSCR, + [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [SWAP] = SWAP, + [TO_BOOL] = TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, + [TO_BOOL_BOOL] = TO_BOOL, + [TO_BOOL_INT] = TO_BOOL, + [TO_BOOL_LIST] = TO_BOOL, + [TO_BOOL_NONE] = TO_BOOL, + [TO_BOOL_STR] = TO_BOOL, + [UNARY_INVERT] = UNARY_INVERT, + [UNARY_NEGATIVE] = UNARY_NEGATIVE, + [UNARY_NOT] = UNARY_NOT, + [UNPACK_EX] = UNPACK_EX, + [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, + [WITH_EXCEPT_START] = WITH_EXCEPT_START, + [YIELD_VALUE] = YIELD_VALUE, +}; +#endif // NEED_OPCODE_METADATA + +#define EXTRA_CASES \ + case 188: \ + case 189: \ + case 190: \ + case 191: \ + case 192: \ + case 193: \ + case 194: \ + case 195: \ + case 196: \ + case 197: \ + case 198: \ + case 199: \ + case 200: \ + case 201: \ + case 202: \ + case 203: \ + case 204: \ + case 205: \ + case 206: \ + case 207: \ + case 208: \ + case 209: \ + case 210: \ + case 211: \ + case 212: \ + case 213: \ + case 214: \ + case 215: \ + case 216: \ + case 217: \ + case 218: \ + case 219: \ + case 220: \ + case 221: \ + case 222: \ + case 223: \ + case 224: \ + case 225: \ + case 226: \ + case 227: \ + case 228: \ + case 229: \ + case 230: \ + case 231: \ + case 232: \ + case 233: \ + case 234: \ + case 235: \ + case 236: \ + case 255: \ + ; + diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 2d9d24cca4542f..cd43716415d1db 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -1,4 +1,7 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py +// This file is generated by Tools/cases_generator/generate_cases.py +// from: +// Python/bytecodes.c +// Do not edit! #ifndef Py_OPCODE_IDS_H #define Py_OPCODE_IDS_H @@ -6,228 +9,227 @@ extern "C" { #endif - /* Instruction opcodes for compiled code */ #define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define INTERPRETER_EXIT 3 -#define END_FOR 4 -#define END_SEND 5 -#define TO_BOOL 6 -#define NOP 9 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define EXIT_INIT_CHECK 16 +#define BEFORE_ASYNC_WITH 1 +#define BEFORE_WITH 2 +#define BINARY_OP_ADD_FLOAT 3 +#define BINARY_OP_ADD_INT 4 +#define BINARY_OP_ADD_UNICODE 5 +#define BINARY_OP_INPLACE_ADD_UNICODE 6 +#define BINARY_OP_MULTIPLY_FLOAT 7 +#define BINARY_OP_MULTIPLY_INT 8 +#define BINARY_OP_SUBTRACT_FLOAT 9 +#define BINARY_OP_SUBTRACT_INT 10 +#define BINARY_SLICE 11 +#define BINARY_SUBSCR 12 +#define BINARY_SUBSCR_DICT 13 +#define BINARY_SUBSCR_GETITEM 14 +#define BINARY_SUBSCR_LIST_INT 15 +#define BINARY_SUBSCR_STR_INT 16 #define RESERVED 17 -#define MAKE_FUNCTION 24 -#define BINARY_SUBSCR 25 -#define BINARY_SLICE 26 -#define STORE_SLICE 27 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define CHECK_EXC_MATCH 36 -#define CHECK_EG_MATCH 37 -#define FORMAT_SIMPLE 40 -#define FORMAT_WITH_SPEC 41 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define CLEANUP_THROW 55 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define RETURN_VALUE 83 -#define SETUP_ANNOTATIONS 85 -#define LOAD_LOCALS 87 -#define POP_EXCEPT 89 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define RETURN_CONST 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define LOAD_FAST_CHECK 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define BUILD_SLICE 133 -#define JUMP_BACKWARD_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define JUMP_BACKWARD 140 -#define LOAD_SUPER_ATTR 141 -#define CALL_FUNCTION_EX 142 -#define LOAD_FAST_AND_CLEAR 143 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define COPY_FREE_VARS 149 -#define YIELD_VALUE 150 -#define RESUME 151 -#define MATCH_CLASS 152 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define CONVERT_VALUE 158 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define LOAD_FAST_LOAD_FAST 168 -#define STORE_FAST_LOAD_FAST 169 -#define STORE_FAST_STORE_FAST 170 -#define CALL 171 -#define KW_NAMES 172 -#define CALL_INTRINSIC_1 173 -#define CALL_INTRINSIC_2 174 -#define LOAD_FROM_DICT_OR_GLOBALS 175 -#define LOAD_FROM_DICT_OR_DEREF 176 -#define SET_FUNCTION_ATTRIBUTE 177 -#define ENTER_EXECUTOR 230 +#define BINARY_SUBSCR_TUPLE_INT 18 +#define CHECK_EG_MATCH 19 +#define CHECK_EXC_MATCH 20 +#define CLEANUP_THROW 21 +#define DELETE_SUBSCR 22 +#define END_ASYNC_FOR 23 +#define END_FOR 24 +#define END_SEND 25 +#define EXIT_INIT_CHECK 26 +#define FORMAT_SIMPLE 27 +#define FORMAT_WITH_SPEC 28 +#define GET_AITER 29 +#define GET_ANEXT 30 +#define GET_ITER 31 +#define GET_LEN 32 +#define GET_YIELD_FROM_ITER 33 +#define INTERPRETER_EXIT 34 +#define LOAD_ASSERTION_ERROR 35 +#define LOAD_BUILD_CLASS 36 +#define LOAD_LOCALS 37 +#define MAKE_FUNCTION 38 +#define MATCH_KEYS 39 +#define MATCH_MAPPING 40 +#define MATCH_SEQUENCE 41 +#define NOP 42 +#define POP_EXCEPT 43 +#define POP_TOP 44 +#define PUSH_EXC_INFO 45 +#define PUSH_NULL 46 +#define RETURN_GENERATOR 47 +#define RETURN_VALUE 48 +#define SETUP_ANNOTATIONS 49 +#define STORE_ATTR_INSTANCE_VALUE 50 +#define STORE_ATTR_SLOT 51 +#define STORE_SLICE 52 +#define STORE_SUBSCR 53 +#define STORE_SUBSCR_DICT 54 +#define STORE_SUBSCR_LIST_INT 55 +#define TO_BOOL 56 +#define TO_BOOL_ALWAYS_TRUE 57 +#define TO_BOOL_BOOL 58 +#define TO_BOOL_INT 59 +#define TO_BOOL_LIST 60 +#define TO_BOOL_NONE 61 +#define TO_BOOL_STR 62 +#define UNARY_INVERT 63 +#define UNARY_NEGATIVE 64 +#define UNARY_NOT 65 +#define WITH_EXCEPT_START 66 +#define HAVE_ARGUMENT 67 +#define BINARY_OP 67 +#define BUILD_CONST_KEY_MAP 68 +#define BUILD_LIST 69 +#define BUILD_MAP 70 +#define BUILD_SET 71 +#define BUILD_SLICE 72 +#define BUILD_STRING 73 +#define BUILD_TUPLE 74 +#define CALL 75 +#define CALL_BOUND_METHOD_EXACT_ARGS 76 +#define CALL_BUILTIN_CLASS 77 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 78 +#define CALL_FUNCTION_EX 79 +#define CALL_INTRINSIC_1 80 +#define CALL_INTRINSIC_2 81 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 82 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 83 +#define CALL_NO_KW_BUILTIN_FAST 84 +#define CALL_NO_KW_BUILTIN_O 85 +#define CALL_NO_KW_ISINSTANCE 86 +#define CALL_NO_KW_LEN 87 +#define CALL_NO_KW_LIST_APPEND 88 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 89 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 90 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 91 +#define CALL_NO_KW_STR_1 92 +#define CALL_NO_KW_TUPLE_1 93 +#define CALL_NO_KW_TYPE_1 94 +#define CALL_PY_EXACT_ARGS 95 +#define CALL_PY_WITH_DEFAULTS 96 +#define COMPARE_OP 97 +#define COMPARE_OP_FLOAT 98 +#define COMPARE_OP_INT 99 +#define COMPARE_OP_STR 100 +#define CONTAINS_OP 101 +#define CONVERT_VALUE 102 +#define COPY 103 +#define COPY_FREE_VARS 104 +#define DELETE_ATTR 105 +#define DELETE_DEREF 106 +#define DELETE_FAST 107 +#define DELETE_GLOBAL 108 +#define DELETE_NAME 109 +#define DICT_MERGE 110 +#define DICT_UPDATE 111 +#define ENTER_EXECUTOR 112 +#define EXTENDED_ARG 113 +#define FOR_ITER 114 +#define FOR_ITER_GEN 115 +#define FOR_ITER_LIST 116 +#define FOR_ITER_RANGE 117 +#define FOR_ITER_TUPLE 118 +#define GET_AWAITABLE 119 +#define IMPORT_FROM 120 +#define IMPORT_NAME 121 +#define IS_OP 122 +#define JUMP_BACKWARD 123 +#define JUMP_BACKWARD_NO_INTERRUPT 124 +#define JUMP_FORWARD 125 +#define KW_NAMES 126 +#define LIST_APPEND 127 +#define LIST_EXTEND 128 +#define LOAD_ATTR 129 +#define LOAD_ATTR_CLASS 130 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 131 +#define LOAD_ATTR_INSTANCE_VALUE 132 +#define LOAD_ATTR_METHOD_LAZY_DICT 133 +#define LOAD_ATTR_METHOD_NO_DICT 134 +#define LOAD_ATTR_METHOD_WITH_VALUES 135 +#define LOAD_ATTR_MODULE 136 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 137 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 138 +#define LOAD_ATTR_PROPERTY 139 +#define LOAD_ATTR_SLOT 140 +#define LOAD_ATTR_WITH_HINT 141 +#define LOAD_CONST 142 +#define LOAD_DEREF 143 +#define LOAD_FAST 144 +#define LOAD_FAST_AND_CLEAR 145 +#define LOAD_FAST_CHECK 146 +#define LOAD_FAST_LOAD_FAST 147 +#define LOAD_FROM_DICT_OR_DEREF 148 +#define LOAD_FROM_DICT_OR_GLOBALS 149 +#define LOAD_GLOBAL 150 +#define LOAD_GLOBAL_BUILTIN 151 +#define LOAD_GLOBAL_MODULE 152 +#define LOAD_NAME 153 +#define LOAD_SUPER_ATTR 154 +#define LOAD_SUPER_ATTR_ATTR 155 +#define LOAD_SUPER_ATTR_METHOD 156 +#define MAKE_CELL 157 +#define MAP_ADD 158 +#define MATCH_CLASS 159 +#define POP_JUMP_IF_FALSE 160 +#define POP_JUMP_IF_NONE 161 +#define POP_JUMP_IF_NOT_NONE 162 +#define POP_JUMP_IF_TRUE 163 +#define RAISE_VARARGS 164 +#define RERAISE 165 +#define RESUME 166 +#define RETURN_CONST 167 +#define SEND 168 +#define SEND_GEN 169 +#define SET_ADD 170 +#define SET_FUNCTION_ATTRIBUTE 171 +#define SET_UPDATE 172 +#define STORE_ATTR 173 +#define STORE_ATTR_WITH_HINT 174 +#define STORE_DEREF 175 +#define STORE_FAST 176 +#define STORE_FAST_LOAD_FAST 177 +#define STORE_FAST_STORE_FAST 178 +#define STORE_GLOBAL 179 +#define STORE_NAME 180 +#define SWAP 181 +#define UNPACK_EX 182 +#define UNPACK_SEQUENCE 183 +#define UNPACK_SEQUENCE_LIST 184 +#define UNPACK_SEQUENCE_TUPLE 185 +#define UNPACK_SEQUENCE_TWO_TUPLE 186 +#define YIELD_VALUE 187 #define MIN_INSTRUMENTED_OPCODE 237 -#define INSTRUMENTED_LOAD_SUPER_ATTR 237 -#define INSTRUMENTED_POP_JUMP_IF_NONE 238 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 -#define INSTRUMENTED_RESUME 240 -#define INSTRUMENTED_CALL 241 -#define INSTRUMENTED_RETURN_VALUE 242 -#define INSTRUMENTED_YIELD_VALUE 243 -#define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_JUMP_FORWARD 245 -#define INSTRUMENTED_JUMP_BACKWARD 246 -#define INSTRUMENTED_RETURN_CONST 247 -#define INSTRUMENTED_FOR_ITER 248 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_RESUME 237 +#define INSTRUMENTED_END_FOR 238 +#define INSTRUMENTED_END_SEND 239 +#define INSTRUMENTED_RETURN_VALUE 240 +#define INSTRUMENTED_RETURN_CONST 241 +#define INSTRUMENTED_YIELD_VALUE 242 +#define INSTRUMENTED_LOAD_SUPER_ATTR 243 +#define INSTRUMENTED_FOR_ITER 244 +#define INSTRUMENTED_CALL 245 +#define INSTRUMENTED_CALL_FUNCTION_EX 246 +#define INSTRUMENTED_INSTRUCTION 247 +#define INSTRUMENTED_JUMP_FORWARD 248 +#define INSTRUMENTED_JUMP_BACKWARD 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_END_FOR 251 -#define INSTRUMENTED_END_SEND 252 -#define INSTRUMENTED_INSTRUCTION 253 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 +#define INSTRUMENTED_POP_JUMP_IF_NONE 252 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 #define INSTRUMENTED_LINE 254 -#define SETUP_FINALLY 256 -#define SETUP_CLEANUP 257 -#define SETUP_WITH 258 -#define POP_BLOCK 259 -#define JUMP 260 -#define JUMP_NO_INTERRUPT 261 -#define LOAD_METHOD 262 -#define LOAD_SUPER_METHOD 263 -#define LOAD_ZERO_SUPER_METHOD 264 -#define LOAD_ZERO_SUPER_ATTR 265 -#define STORE_FAST_MAYBE_NULL 266 -#define LOAD_CLOSURE 267 -#define TO_BOOL_ALWAYS_TRUE 7 -#define TO_BOOL_BOOL 8 -#define TO_BOOL_INT 10 -#define TO_BOOL_LIST 13 -#define TO_BOOL_NONE 14 -#define TO_BOOL_STR 18 -#define BINARY_OP_MULTIPLY_INT 19 -#define BINARY_OP_ADD_INT 20 -#define BINARY_OP_SUBTRACT_INT 21 -#define BINARY_OP_MULTIPLY_FLOAT 22 -#define BINARY_OP_ADD_FLOAT 23 -#define BINARY_OP_SUBTRACT_FLOAT 28 -#define BINARY_OP_ADD_UNICODE 29 -#define BINARY_OP_INPLACE_ADD_UNICODE 34 -#define BINARY_SUBSCR_DICT 38 -#define BINARY_SUBSCR_GETITEM 39 -#define BINARY_SUBSCR_LIST_INT 42 -#define BINARY_SUBSCR_STR_INT 43 -#define BINARY_SUBSCR_TUPLE_INT 44 -#define STORE_SUBSCR_DICT 45 -#define STORE_SUBSCR_LIST_INT 46 -#define SEND_GEN 47 -#define UNPACK_SEQUENCE_TWO_TUPLE 48 -#define UNPACK_SEQUENCE_TUPLE 56 -#define UNPACK_SEQUENCE_LIST 57 -#define STORE_ATTR_INSTANCE_VALUE 58 -#define STORE_ATTR_SLOT 59 -#define STORE_ATTR_WITH_HINT 62 -#define LOAD_GLOBAL_MODULE 63 -#define LOAD_GLOBAL_BUILTIN 64 -#define LOAD_SUPER_ATTR_ATTR 65 -#define LOAD_SUPER_ATTR_METHOD 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 70 -#define LOAD_ATTR_WITH_HINT 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_CLASS 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_LAZY_DICT 81 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 -#define COMPARE_OP_FLOAT 86 -#define COMPARE_OP_INT 88 -#define COMPARE_OP_STR 111 -#define FOR_ITER_LIST 112 -#define FOR_ITER_TUPLE 113 -#define FOR_ITER_RANGE 132 -#define FOR_ITER_GEN 136 -#define CALL_BOUND_METHOD_EXACT_ARGS 148 -#define CALL_PY_EXACT_ARGS 153 -#define CALL_PY_WITH_DEFAULTS 154 -#define CALL_NO_KW_TYPE_1 155 -#define CALL_NO_KW_STR_1 159 -#define CALL_NO_KW_TUPLE_1 160 -#define CALL_BUILTIN_CLASS 161 -#define CALL_NO_KW_BUILTIN_O 166 -#define CALL_NO_KW_BUILTIN_FAST 167 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 -#define CALL_NO_KW_LEN 179 -#define CALL_NO_KW_ISINSTANCE 180 -#define CALL_NO_KW_LIST_APPEND 181 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 - +#define JUMP 256 +#define JUMP_NO_INTERRUPT 257 +#define LOAD_CLOSURE 258 +#define LOAD_METHOD 259 +#define LOAD_SUPER_METHOD 260 +#define LOAD_ZERO_SUPER_ATTR 261 +#define LOAD_ZERO_SUPER_METHOD 262 +#define POP_BLOCK 263 +#define SETUP_CLEANUP 264 +#define SETUP_FINALLY 265 +#define SETUP_WITH 266 +#define STORE_FAST_MAYBE_NULL 267 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 17101d1d94757b..b02aa771c347e7 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -103,4 +103,228 @@ # An irregular case: _specializations["BINARY_OP"].append("BINARY_OP_INPLACE_ADD_UNICODE") -_specialized_instructions = [opcode for family in _specializations.values() for opcode in family] +_specialized_opmap = { + 'BINARY_OP_ADD_FLOAT': 3, + 'BINARY_OP_ADD_INT': 4, + 'BINARY_OP_ADD_UNICODE': 5, + 'BINARY_OP_INPLACE_ADD_UNICODE': 6, + 'BINARY_OP_MULTIPLY_FLOAT': 7, + 'BINARY_OP_MULTIPLY_INT': 8, + 'BINARY_OP_SUBTRACT_FLOAT': 9, + 'BINARY_OP_SUBTRACT_INT': 10, + 'BINARY_SUBSCR_DICT': 13, + 'BINARY_SUBSCR_GETITEM': 14, + 'BINARY_SUBSCR_LIST_INT': 15, + 'BINARY_SUBSCR_STR_INT': 16, + 'BINARY_SUBSCR_TUPLE_INT': 18, + 'STORE_ATTR_INSTANCE_VALUE': 50, + 'STORE_ATTR_SLOT': 51, + 'STORE_SUBSCR_DICT': 54, + 'STORE_SUBSCR_LIST_INT': 55, + 'TO_BOOL_ALWAYS_TRUE': 57, + 'TO_BOOL_BOOL': 58, + 'TO_BOOL_INT': 59, + 'TO_BOOL_LIST': 60, + 'TO_BOOL_NONE': 61, + 'TO_BOOL_STR': 62, + 'CALL_BOUND_METHOD_EXACT_ARGS': 76, + 'CALL_BUILTIN_CLASS': 77, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 78, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 82, + 'CALL_NO_KW_ALLOC_AND_ENTER_INIT': 83, + 'CALL_NO_KW_BUILTIN_FAST': 84, + 'CALL_NO_KW_BUILTIN_O': 85, + 'CALL_NO_KW_ISINSTANCE': 86, + 'CALL_NO_KW_LEN': 87, + 'CALL_NO_KW_LIST_APPEND': 88, + 'CALL_NO_KW_METHOD_DESCRIPTOR_FAST': 89, + 'CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS': 90, + 'CALL_NO_KW_METHOD_DESCRIPTOR_O': 91, + 'CALL_NO_KW_STR_1': 92, + 'CALL_NO_KW_TUPLE_1': 93, + 'CALL_NO_KW_TYPE_1': 94, + 'CALL_PY_EXACT_ARGS': 95, + 'CALL_PY_WITH_DEFAULTS': 96, + 'COMPARE_OP_FLOAT': 98, + 'COMPARE_OP_INT': 99, + 'COMPARE_OP_STR': 100, + 'FOR_ITER_GEN': 115, + 'FOR_ITER_LIST': 116, + 'FOR_ITER_RANGE': 117, + 'FOR_ITER_TUPLE': 118, + 'LOAD_ATTR_CLASS': 130, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 131, + 'LOAD_ATTR_INSTANCE_VALUE': 132, + 'LOAD_ATTR_METHOD_LAZY_DICT': 133, + 'LOAD_ATTR_METHOD_NO_DICT': 134, + 'LOAD_ATTR_METHOD_WITH_VALUES': 135, + 'LOAD_ATTR_MODULE': 136, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 137, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 138, + 'LOAD_ATTR_PROPERTY': 139, + 'LOAD_ATTR_SLOT': 140, + 'LOAD_ATTR_WITH_HINT': 141, + 'LOAD_GLOBAL_BUILTIN': 151, + 'LOAD_GLOBAL_MODULE': 152, + 'LOAD_SUPER_ATTR_ATTR': 155, + 'LOAD_SUPER_ATTR_METHOD': 156, + 'SEND_GEN': 169, + 'STORE_ATTR_WITH_HINT': 174, + 'UNPACK_SEQUENCE_LIST': 184, + 'UNPACK_SEQUENCE_TUPLE': 185, + 'UNPACK_SEQUENCE_TWO_TUPLE': 186, +} + +opmap = { + 'CACHE': 0, + 'BEFORE_ASYNC_WITH': 1, + 'BEFORE_WITH': 2, + 'BINARY_SLICE': 11, + 'BINARY_SUBSCR': 12, + 'RESERVED': 17, + 'CHECK_EG_MATCH': 19, + 'CHECK_EXC_MATCH': 20, + 'CLEANUP_THROW': 21, + 'DELETE_SUBSCR': 22, + 'END_ASYNC_FOR': 23, + 'END_FOR': 24, + 'END_SEND': 25, + 'EXIT_INIT_CHECK': 26, + 'FORMAT_SIMPLE': 27, + 'FORMAT_WITH_SPEC': 28, + 'GET_AITER': 29, + 'GET_ANEXT': 30, + 'GET_ITER': 31, + 'GET_LEN': 32, + 'GET_YIELD_FROM_ITER': 33, + 'INTERPRETER_EXIT': 34, + 'LOAD_ASSERTION_ERROR': 35, + 'LOAD_BUILD_CLASS': 36, + 'LOAD_LOCALS': 37, + 'MAKE_FUNCTION': 38, + 'MATCH_KEYS': 39, + 'MATCH_MAPPING': 40, + 'MATCH_SEQUENCE': 41, + 'NOP': 42, + 'POP_EXCEPT': 43, + 'POP_TOP': 44, + 'PUSH_EXC_INFO': 45, + 'PUSH_NULL': 46, + 'RETURN_GENERATOR': 47, + 'RETURN_VALUE': 48, + 'SETUP_ANNOTATIONS': 49, + 'STORE_SLICE': 52, + 'STORE_SUBSCR': 53, + 'TO_BOOL': 56, + 'UNARY_INVERT': 63, + 'UNARY_NEGATIVE': 64, + 'UNARY_NOT': 65, + 'WITH_EXCEPT_START': 66, + 'BINARY_OP': 67, + 'BUILD_CONST_KEY_MAP': 68, + 'BUILD_LIST': 69, + 'BUILD_MAP': 70, + 'BUILD_SET': 71, + 'BUILD_SLICE': 72, + 'BUILD_STRING': 73, + 'BUILD_TUPLE': 74, + 'CALL': 75, + 'CALL_FUNCTION_EX': 79, + 'CALL_INTRINSIC_1': 80, + 'CALL_INTRINSIC_2': 81, + 'COMPARE_OP': 97, + 'CONTAINS_OP': 101, + 'CONVERT_VALUE': 102, + 'COPY': 103, + 'COPY_FREE_VARS': 104, + 'DELETE_ATTR': 105, + 'DELETE_DEREF': 106, + 'DELETE_FAST': 107, + 'DELETE_GLOBAL': 108, + 'DELETE_NAME': 109, + 'DICT_MERGE': 110, + 'DICT_UPDATE': 111, + 'ENTER_EXECUTOR': 112, + 'EXTENDED_ARG': 113, + 'FOR_ITER': 114, + 'GET_AWAITABLE': 119, + 'IMPORT_FROM': 120, + 'IMPORT_NAME': 121, + 'IS_OP': 122, + 'JUMP_BACKWARD': 123, + 'JUMP_BACKWARD_NO_INTERRUPT': 124, + 'JUMP_FORWARD': 125, + 'KW_NAMES': 126, + 'LIST_APPEND': 127, + 'LIST_EXTEND': 128, + 'LOAD_ATTR': 129, + 'LOAD_CONST': 142, + 'LOAD_DEREF': 143, + 'LOAD_FAST': 144, + 'LOAD_FAST_AND_CLEAR': 145, + 'LOAD_FAST_CHECK': 146, + 'LOAD_FAST_LOAD_FAST': 147, + 'LOAD_FROM_DICT_OR_DEREF': 148, + 'LOAD_FROM_DICT_OR_GLOBALS': 149, + 'LOAD_GLOBAL': 150, + 'LOAD_NAME': 153, + 'LOAD_SUPER_ATTR': 154, + 'MAKE_CELL': 157, + 'MAP_ADD': 158, + 'MATCH_CLASS': 159, + 'POP_JUMP_IF_FALSE': 160, + 'POP_JUMP_IF_NONE': 161, + 'POP_JUMP_IF_NOT_NONE': 162, + 'POP_JUMP_IF_TRUE': 163, + 'RAISE_VARARGS': 164, + 'RERAISE': 165, + 'RESUME': 166, + 'RETURN_CONST': 167, + 'SEND': 168, + 'SET_ADD': 170, + 'SET_FUNCTION_ATTRIBUTE': 171, + 'SET_UPDATE': 172, + 'STORE_ATTR': 173, + 'STORE_DEREF': 175, + 'STORE_FAST': 176, + 'STORE_FAST_LOAD_FAST': 177, + 'STORE_FAST_STORE_FAST': 178, + 'STORE_GLOBAL': 179, + 'STORE_NAME': 180, + 'SWAP': 181, + 'UNPACK_EX': 182, + 'UNPACK_SEQUENCE': 183, + 'YIELD_VALUE': 187, + 'INSTRUMENTED_RESUME': 237, + 'INSTRUMENTED_END_FOR': 238, + 'INSTRUMENTED_END_SEND': 239, + 'INSTRUMENTED_RETURN_VALUE': 240, + 'INSTRUMENTED_RETURN_CONST': 241, + 'INSTRUMENTED_YIELD_VALUE': 242, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 243, + 'INSTRUMENTED_FOR_ITER': 244, + 'INSTRUMENTED_CALL': 245, + 'INSTRUMENTED_CALL_FUNCTION_EX': 246, + 'INSTRUMENTED_INSTRUCTION': 247, + 'INSTRUMENTED_JUMP_FORWARD': 248, + 'INSTRUMENTED_JUMP_BACKWARD': 249, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'INSTRUMENTED_LINE': 254, + 'JUMP': 256, + 'JUMP_NO_INTERRUPT': 257, + 'LOAD_CLOSURE': 258, + 'LOAD_METHOD': 259, + 'LOAD_SUPER_METHOD': 260, + 'LOAD_ZERO_SUPER_ATTR': 261, + 'LOAD_ZERO_SUPER_METHOD': 262, + 'POP_BLOCK': 263, + 'SETUP_CLEANUP': 264, + 'SETUP_FINALLY': 265, + 'SETUP_WITH': 266, + 'STORE_FAST_MAYBE_NULL': 267, +} +MIN_INSTRUMENTED_OPCODE = 237 +HAVE_ARGUMENT = 67 diff --git a/Lib/dis.py b/Lib/dis.py index bf1a1e2ff7ac19..4f4e77f32503df 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -14,7 +14,7 @@ _intrinsic_1_descs, _intrinsic_2_descs, _specializations, - _specialized_instructions, + _specialized_opmap, ) __all__ = ["code_info", "dis", "disassemble", "distb", "disco", @@ -49,11 +49,11 @@ _all_opname = list(opname) _all_opmap = dict(opmap) -_empty_slot = [slot for slot, name in enumerate(_all_opname) if name.startswith("<")] -for spec_op, specialized in zip(_empty_slot, _specialized_instructions): +for name, op in _specialized_opmap.items(): # fill opname and opmap - _all_opname[spec_op] = specialized - _all_opmap[specialized] = spec_op + assert op < len(_all_opname) + _all_opname[op] = name + _all_opmap[name] = op deoptmap = { specialized: base for base, family in _specializations.items() for specialized in family diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 5f0d659b1ed535..0717e202ecbcde 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -454,6 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) # Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) # Python 3.13a1 3558 (Reorder the stack items for CALL) +# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) # Python 3.14 will start with 3600 @@ -470,7 +471,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3558).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3559).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 5a9f8ddd0738db..6b9d9ce811a61c 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -6,8 +6,7 @@ # Note that __all__ is further extended below -__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", - "HAVE_ARGUMENT", "EXTENDED_ARG"] +__all__ = ["cmp_op", "stack_effect", "hascompare"] import _opcode from _opcode import stack_effect @@ -15,214 +14,17 @@ import sys # The build uses older versions of Python which do not have _opcode_metadata if sys.version_info[:2] >= (3, 13): - from _opcode_metadata import _specializations, _specialized_instructions + from _opcode_metadata import _specializations, _specialized_opmap + from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE + EXTENDED_ARG = opmap['EXTENDED_ARG'] -cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -opmap = {} - -def def_op(name, op): - opmap[name] = op - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('CACHE', 0) -def_op('POP_TOP', 1) -def_op('PUSH_NULL', 2) -def_op('INTERPRETER_EXIT', 3) -def_op('END_FOR', 4) -def_op('END_SEND', 5) -def_op('TO_BOOL', 6) - -def_op('NOP', 9) - -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) -def_op('EXIT_INIT_CHECK', 16) - -# We reserve 17 as it is the initial value for the specializing counter -# This helps us catch cases where we attempt to execute a cache. -def_op('RESERVED', 17) - -def_op('MAKE_FUNCTION', 24) -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_SLICE', 26) -def_op('STORE_SLICE', 27) - -def_op('GET_LEN', 30) -def_op('MATCH_MAPPING', 31) -def_op('MATCH_SEQUENCE', 32) -def_op('MATCH_KEYS', 33) - -def_op('PUSH_EXC_INFO', 35) -def_op('CHECK_EXC_MATCH', 36) -def_op('CHECK_EG_MATCH', 37) - -def_op('FORMAT_SIMPLE', 40) -def_op('FORMAT_WITH_SPEC', 41) - -def_op('WITH_EXCEPT_START', 49) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) -def_op('BEFORE_WITH', 53) -def_op('END_ASYNC_FOR', 54) -def_op('CLEANUP_THROW', 55) - -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) - -def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) - -def_op('LOAD_BUILD_CLASS', 71) - -def_op('LOAD_ASSERTION_ERROR', 74) -def_op('RETURN_GENERATOR', 75) - -def_op('RETURN_VALUE', 83) + opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] + for op, i in opmap.items(): + opname[i] = op -def_op('SETUP_ANNOTATIONS', 85) + __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"]) -def_op('LOAD_LOCALS', 87) - -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # real opcodes from here have an argument: - -def_op('STORE_NAME', 90) # Index in name list -def_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -def_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -def_op('STORE_ATTR', 95) # Index in name list -def_op('DELETE_ATTR', 96) # "" -def_op('STORE_GLOBAL', 97) # "" -def_op('DELETE_GLOBAL', 98) # "" -def_op('SWAP', 99) -def_op('LOAD_CONST', 100) # Index in const list -def_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries -def_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -def_op('IMPORT_NAME', 108) # Index in name list -def_op('IMPORT_FROM', 109) # Index in name list -def_op('JUMP_FORWARD', 110) # Number of words to skip - -def_op('POP_JUMP_IF_FALSE', 114) -def_op('POP_JUMP_IF_TRUE', 115) -def_op('LOAD_GLOBAL', 116) # Index in name list -def_op('IS_OP', 117) -def_op('CONTAINS_OP', 118) -def_op('RERAISE', 119) -def_op('COPY', 120) -def_op('RETURN_CONST', 121) -def_op('BINARY_OP', 122) -def_op('SEND', 123) # Number of words to skip -def_op('LOAD_FAST', 124) # Local variable number, no null check -def_op('STORE_FAST', 125) # Local variable number -def_op('DELETE_FAST', 126) # Local variable number -def_op('LOAD_FAST_CHECK', 127) # Local variable number -def_op('POP_JUMP_IF_NOT_NONE', 128) -def_op('POP_JUMP_IF_NONE', 129) -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('GET_AWAITABLE', 131) -def_op('BUILD_SLICE', 133) # Number of items -def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) -def_op('MAKE_CELL', 135) -def_op('LOAD_DEREF', 137) -def_op('STORE_DEREF', 138) -def_op('DELETE_DEREF', 139) -def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -def_op('LOAD_SUPER_ATTR', 141) -def_op('CALL_FUNCTION_EX', 142) # Flags -def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = opmap['EXTENDED_ARG'] -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) -def_op('COPY_FREE_VARS', 149) -def_op('YIELD_VALUE', 150) -def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py -def_op('MATCH_CLASS', 152) - -def_op('BUILD_CONST_KEY_MAP', 156) -def_op('BUILD_STRING', 157) -def_op('CONVERT_VALUE', 158) - -def_op('LIST_EXTEND', 162) -def_op('SET_UPDATE', 163) -def_op('DICT_MERGE', 164) -def_op('DICT_UPDATE', 165) - -def_op('LOAD_FAST_LOAD_FAST', 168) -def_op('STORE_FAST_LOAD_FAST', 169) -def_op('STORE_FAST_STORE_FAST', 170) -def_op('CALL', 171) -def_op('KW_NAMES', 172) -def_op('CALL_INTRINSIC_1', 173) -def_op('CALL_INTRINSIC_2', 174) -def_op('LOAD_FROM_DICT_OR_GLOBALS', 175) -def_op('LOAD_FROM_DICT_OR_DEREF', 176) -def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute - -# Optimizer hook -def_op('ENTER_EXECUTOR', 230) - -# Instrumented instructions -MIN_INSTRUMENTED_OPCODE = 237 - -def_op('INSTRUMENTED_LOAD_SUPER_ATTR', 237) -def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) -def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) -def_op('INSTRUMENTED_RESUME', 240) -def_op('INSTRUMENTED_CALL', 241) -def_op('INSTRUMENTED_RETURN_VALUE', 242) -def_op('INSTRUMENTED_YIELD_VALUE', 243) -def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) -def_op('INSTRUMENTED_JUMP_FORWARD', 245) -def_op('INSTRUMENTED_JUMP_BACKWARD', 246) -def_op('INSTRUMENTED_RETURN_CONST', 247) -def_op('INSTRUMENTED_FOR_ITER', 248) -def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) -def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) -def_op('INSTRUMENTED_END_FOR', 251) -def_op('INSTRUMENTED_END_SEND', 252) -def_op('INSTRUMENTED_INSTRUCTION', 253) -def_op('INSTRUMENTED_LINE', 254) -# 255 is reserved - - -# Pseudo ops are above 255: - -def_op('SETUP_FINALLY', 256) -def_op('SETUP_CLEANUP', 257) -def_op('SETUP_WITH', 258) -def_op('POP_BLOCK', 259) - -def_op('JUMP', 260) -def_op('JUMP_NO_INTERRUPT', 261) - -def_op('LOAD_METHOD', 262) -def_op('LOAD_SUPER_METHOD', 263) -def_op('LOAD_ZERO_SUPER_METHOD', 264) -def_op('LOAD_ZERO_SUPER_ATTR', 265) - -def_op('STORE_FAST_MAYBE_NULL', 266) -def_op('LOAD_CLOSURE', 267) - -del def_op - -opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] -for op, i in opmap.items(): - opname[i] = op +cmp_op = ('<', '<=', '==', '!=', '>', '>=') # The build uses older versions of Python which do not have _opcode.has_* functions if sys.version_info[:2] >= (3, 13): @@ -243,7 +45,7 @@ def def_op(name, op): _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() -hascompare = [opmap["COMPARE_OP"]] + hascompare = [opmap["COMPARE_OP"]] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f49c60a01a54ea..aa967178c1dc39 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -196,11 +196,11 @@ def bug42562(): # Extended arg followed by NOP code_bug_45757 = bytes([ - 0x90, 0x01, # EXTENDED_ARG 0x01 - 0x09, 0xFF, # NOP 0xFF - 0x90, 0x01, # EXTENDED_ARG 0x01 - 0x64, 0x29, # LOAD_CONST 0x29 - 0x53, 0x00, # RETURN_VALUE 0x00 + opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 + opcode.opmap['NOP'], 0xFF, # NOP 0xFF + opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 + opcode.opmap['LOAD_CONST'], 0x29, # LOAD_CONST 0x29 + opcode.opmap['RETURN_VALUE'], 0x00, # RETURN_VALUE 0x00 ]) dis_bug_45757 = """\ @@ -931,7 +931,7 @@ def do_disassembly_test(self, func, expected, with_offsets=False): with_offsets) def test_opmap(self): - self.assertEqual(dis.opmap["NOP"], 9) + self.assertEqual(dis.opmap["CACHE"], 0) self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst) self.assertIn(dis.opmap["STORE_NAME"], dis.hasname) @@ -940,7 +940,6 @@ def test_opname(self): def test_boundaries(self): self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG) - self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT) def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', @@ -1617,197 +1616,197 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=135, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=135, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=24, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=1, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=74, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=69, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=70, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=149, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=135, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=135, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=24, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=104, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=74, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=168, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=104, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=147, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=31, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=114, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=125, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=24, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=146, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=67, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=125, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=42, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=67, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=66, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=20, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=None, is_jump_target=False), ] @@ -1943,7 +1942,7 @@ def test_baseopname_and_baseopcode(self): self.assertEqual(code, baseopcode) # Specialized instructions - for name in opcode._specialized_instructions: + for name in opcode._specialized_opmap: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 582392ecddcb91..50c9f61017e022 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -361,7 +361,7 @@ def is_specialized(f): for instruction in dis.get_instructions(f, adaptive=True): opname = instruction.opname if ( - opname in opcode._specialized_instructions + opname in opcode._specialized_opmap # Exclude superinstructions: and "__" not in opname ): diff --git a/Makefile.pre.in b/Makefile.pre.in index 32d928316f2215..bcec0782f6e95e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1431,14 +1431,9 @@ regen-opcode: # using Tools/build/generate_opcode_h.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ - $(srcdir)/Lib/_opcode_metadata.py \ - $(srcdir)/Include/opcode_ids.h.new \ $(srcdir)/Include/opcode.h.new \ - $(srcdir)/Python/opcode_targets.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new - $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new .PHONY: regen-token @@ -1550,12 +1545,16 @@ regen-cases: $(srcdir)/Tools/cases_generator/generate_cases.py \ $(CASESFLAG) \ -o $(srcdir)/Python/generated_cases.c.h.new \ + -n $(srcdir)/Include/opcode_ids.h.new \ + -t $(srcdir)/Python/opcode_targets.h.new \ -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ -e $(srcdir)/Python/executor_cases.c.h.new \ -p $(srcdir)/Lib/_opcode_metadata.py.new \ -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Python/abstract_interp_cases.c.h $(srcdir)/Python/abstract_interp_cases.c.h.new diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst new file mode 100644 index 00000000000000..dc10f672d8871e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst @@ -0,0 +1,2 @@ +Opcode IDs are generated from bytecodes.c instead of being hard coded in +opcode.py. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 6987a2382d81c2..2c9c8cec77ff9f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -6,7 +6,8 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs -#include "pycore_opcode.h" // _PyOpcode_Deopt +#include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 17571535048e23..80e118e8a8aa93 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -7,6 +7,8 @@ #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt + #include "frameobject.h" // PyFrameObject #include "pycore_frame.h" diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index c1189c883b667c..ed02f9163549b9 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -14,7 +14,7 @@ -C <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" /> + <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 0dca507e28bc14..3fd6cdade69f9e 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,17 +1,17 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,164,0,0,0,151,0,100,0,100,1, - 108,0,90,0,100,0,100,1,108,1,90,1,101,2,2,0, - 100,2,171,1,0,0,0,0,0,0,1,0,101,2,2,0, - 100,3,101,0,106,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,171,2,0,0,0,0,0,0, - 1,0,101,1,106,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,2,0,171,0,0,0,0,0, - 0,0,100,4,25,0,0,0,90,5,100,5,68,0,93,20, - 0,0,90,6,101,2,2,0,100,6,101,6,40,0,100,7, - 101,5,101,6,25,0,0,0,40,0,157,4,171,1,0,0, - 0,0,0,0,1,0,140,22,0,0,4,0,121,1,41,8, + 0,0,0,0,0,243,164,0,0,0,166,0,142,0,142,1, + 121,0,180,0,142,0,142,1,121,1,180,1,153,2,46,0, + 142,2,75,1,0,0,0,0,0,0,44,0,153,2,46,0, + 142,3,153,0,129,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,75,2,0,0,0,0,0,0, + 44,0,153,1,129,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,46,0,75,0,0,0,0,0, + 0,0,142,4,12,0,0,0,180,5,142,5,31,0,114,20, + 0,0,180,6,153,2,46,0,142,6,153,6,27,0,142,7, + 153,5,153,6,12,0,0,0,27,0,73,4,75,1,0,0, + 0,0,0,0,44,0,123,22,0,0,24,0,167,1,41,8, 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, 101,108,108,111,32,87,111,114,108,100,122,8,115,121,115,46, 97,114,103,118,218,6,99,111,110,102,105,103,41,5,218,12, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e9a5cf59e7d689..2a5ad2c942fb3e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -17,7 +17,6 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_opcode.h" // EXTRA_CASES #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 210c37b37225bb..305eb0bfe2a7c4 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,191 +1,192 @@ static void *opcode_targets[256] = { &&TARGET_CACHE, - &&TARGET_POP_TOP, - &&TARGET_PUSH_NULL, - &&TARGET_INTERPRETER_EXIT, - &&TARGET_END_FOR, - &&TARGET_END_SEND, - &&TARGET_TO_BOOL, - &&TARGET_TO_BOOL_ALWAYS_TRUE, - &&TARGET_TO_BOOL_BOOL, - &&TARGET_NOP, - &&TARGET_TO_BOOL_INT, - &&TARGET_UNARY_NEGATIVE, - &&TARGET_UNARY_NOT, - &&TARGET_TO_BOOL_LIST, - &&TARGET_TO_BOOL_NONE, - &&TARGET_UNARY_INVERT, - &&TARGET_EXIT_INIT_CHECK, - &&TARGET_RESERVED, - &&TARGET_TO_BOOL_STR, - &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_ADD_INT, - &&TARGET_BINARY_OP_SUBTRACT_INT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BEFORE_ASYNC_WITH, + &&TARGET_BEFORE_WITH, &&TARGET_BINARY_OP_ADD_FLOAT, - &&TARGET_MAKE_FUNCTION, - &&TARGET_BINARY_SUBSCR, - &&TARGET_BINARY_SLICE, - &&TARGET_STORE_SLICE, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, - &&TARGET_GET_LEN, - &&TARGET_MATCH_MAPPING, - &&TARGET_MATCH_SEQUENCE, - &&TARGET_MATCH_KEYS, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_PUSH_EXC_INFO, - &&TARGET_CHECK_EXC_MATCH, - &&TARGET_CHECK_EG_MATCH, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_BINARY_SLICE, + &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, - &&TARGET_FORMAT_SIMPLE, - &&TARGET_FORMAT_WITH_SPEC, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_STR_INT, + &&TARGET_RESERVED, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_STORE_SUBSCR_DICT, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_SEND_GEN, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_WITH_EXCEPT_START, - &&TARGET_GET_AITER, - &&TARGET_GET_ANEXT, - &&TARGET_BEFORE_ASYNC_WITH, - &&TARGET_BEFORE_WITH, - &&TARGET_END_ASYNC_FOR, + &&TARGET_CHECK_EG_MATCH, + &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_SUPER_ATTR_ATTR, - &&TARGET_LOAD_SUPER_ATTR_METHOD, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_END_ASYNC_FOR, + &&TARGET_END_FOR, + &&TARGET_END_SEND, + &&TARGET_EXIT_INIT_CHECK, + &&TARGET_FORMAT_SIMPLE, + &&TARGET_FORMAT_WITH_SPEC, + &&TARGET_GET_AITER, + &&TARGET_GET_ANEXT, &&TARGET_GET_ITER, + &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_LOCALS, + &&TARGET_MAKE_FUNCTION, + &&TARGET_MATCH_KEYS, + &&TARGET_MATCH_MAPPING, + &&TARGET_MATCH_SEQUENCE, + &&TARGET_NOP, + &&TARGET_POP_EXCEPT, + &&TARGET_POP_TOP, + &&TARGET_PUSH_EXC_INFO, + &&TARGET_PUSH_NULL, &&TARGET_RETURN_GENERATOR, - &&TARGET_LOAD_ATTR_CLASS, - &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_COMPARE_OP_FLOAT, - &&TARGET_LOAD_LOCALS, - &&TARGET_COMPARE_OP_INT, - &&TARGET_POP_EXCEPT, - &&TARGET_STORE_NAME, - &&TARGET_DELETE_NAME, - &&TARGET_UNPACK_SEQUENCE, - &&TARGET_FOR_ITER, - &&TARGET_UNPACK_EX, - &&TARGET_STORE_ATTR, - &&TARGET_DELETE_ATTR, - &&TARGET_STORE_GLOBAL, - &&TARGET_DELETE_GLOBAL, - &&TARGET_SWAP, - &&TARGET_LOAD_CONST, - &&TARGET_LOAD_NAME, - &&TARGET_BUILD_TUPLE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_SLICE, + &&TARGET_STORE_SUBSCR, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_TO_BOOL, + &&TARGET_TO_BOOL_ALWAYS_TRUE, + &&TARGET_TO_BOOL_BOOL, + &&TARGET_TO_BOOL_INT, + &&TARGET_TO_BOOL_LIST, + &&TARGET_TO_BOOL_NONE, + &&TARGET_TO_BOOL_STR, + &&TARGET_UNARY_INVERT, + &&TARGET_UNARY_NEGATIVE, + &&TARGET_UNARY_NOT, + &&TARGET_WITH_EXCEPT_START, + &&TARGET_BINARY_OP, + &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_LIST, - &&TARGET_BUILD_SET, &&TARGET_BUILD_MAP, - &&TARGET_LOAD_ATTR, + &&TARGET_BUILD_SET, + &&TARGET_BUILD_SLICE, + &&TARGET_BUILD_STRING, + &&TARGET_BUILD_TUPLE, + &&TARGET_CALL, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_CALL_INTRINSIC_1, + &&TARGET_CALL_INTRINSIC_2, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_NO_KW_BUILTIN_O, + &&TARGET_CALL_NO_KW_ISINSTANCE, + &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_NO_KW_LIST_APPEND, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, + &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_COMPARE_OP, - &&TARGET_IMPORT_NAME, - &&TARGET_IMPORT_FROM, - &&TARGET_JUMP_FORWARD, + &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, - &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_TUPLE, - &&TARGET_POP_JUMP_IF_FALSE, - &&TARGET_POP_JUMP_IF_TRUE, - &&TARGET_LOAD_GLOBAL, - &&TARGET_IS_OP, &&TARGET_CONTAINS_OP, - &&TARGET_RERAISE, + &&TARGET_CONVERT_VALUE, &&TARGET_COPY, - &&TARGET_RETURN_CONST, - &&TARGET_BINARY_OP, - &&TARGET_SEND, - &&TARGET_LOAD_FAST, - &&TARGET_STORE_FAST, + &&TARGET_COPY_FREE_VARS, + &&TARGET_DELETE_ATTR, + &&TARGET_DELETE_DEREF, &&TARGET_DELETE_FAST, - &&TARGET_LOAD_FAST_CHECK, - &&TARGET_POP_JUMP_IF_NOT_NONE, - &&TARGET_POP_JUMP_IF_NONE, - &&TARGET_RAISE_VARARGS, - &&TARGET_GET_AWAITABLE, + &&TARGET_DELETE_GLOBAL, + &&TARGET_DELETE_NAME, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, + &&TARGET_ENTER_EXECUTOR, + &&TARGET_EXTENDED_ARG, + &&TARGET_FOR_ITER, + &&TARGET_FOR_ITER_GEN, + &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, - &&TARGET_BUILD_SLICE, + &&TARGET_FOR_ITER_TUPLE, + &&TARGET_GET_AWAITABLE, + &&TARGET_IMPORT_FROM, + &&TARGET_IMPORT_NAME, + &&TARGET_IS_OP, + &&TARGET_JUMP_BACKWARD, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, - &&TARGET_MAKE_CELL, - &&TARGET_FOR_ITER_GEN, + &&TARGET_JUMP_FORWARD, + &&TARGET_KW_NAMES, + &&TARGET_LIST_APPEND, + &&TARGET_LIST_EXTEND, + &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_CONST, &&TARGET_LOAD_DEREF, - &&TARGET_STORE_DEREF, - &&TARGET_DELETE_DEREF, - &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_SUPER_ATTR, - &&TARGET_CALL_FUNCTION_EX, + &&TARGET_LOAD_FAST, &&TARGET_LOAD_FAST_AND_CLEAR, - &&TARGET_EXTENDED_ARG, - &&TARGET_LIST_APPEND, - &&TARGET_SET_ADD, + &&TARGET_LOAD_FAST_CHECK, + &&TARGET_LOAD_FAST_LOAD_FAST, + &&TARGET_LOAD_FROM_DICT_OR_DEREF, + &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, + &&TARGET_LOAD_GLOBAL, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_NAME, + &&TARGET_LOAD_SUPER_ATTR, + &&TARGET_LOAD_SUPER_ATTR_ATTR, + &&TARGET_LOAD_SUPER_ATTR_METHOD, + &&TARGET_MAKE_CELL, &&TARGET_MAP_ADD, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_COPY_FREE_VARS, - &&TARGET_YIELD_VALUE, - &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_BUILD_CONST_KEY_MAP, - &&TARGET_BUILD_STRING, - &&TARGET_CONVERT_VALUE, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_LIST_EXTEND, + &&TARGET_POP_JUMP_IF_FALSE, + &&TARGET_POP_JUMP_IF_NONE, + &&TARGET_POP_JUMP_IF_NOT_NONE, + &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_RAISE_VARARGS, + &&TARGET_RERAISE, + &&TARGET_RESUME, + &&TARGET_RETURN_CONST, + &&TARGET_SEND, + &&TARGET_SEND_GEN, + &&TARGET_SET_ADD, + &&TARGET_SET_FUNCTION_ATTRIBUTE, &&TARGET_SET_UPDATE, - &&TARGET_DICT_MERGE, - &&TARGET_DICT_UPDATE, - &&TARGET_CALL_NO_KW_BUILTIN_O, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, - &&TARGET_LOAD_FAST_LOAD_FAST, + &&TARGET_STORE_ATTR, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_DEREF, + &&TARGET_STORE_FAST, &&TARGET_STORE_FAST_LOAD_FAST, &&TARGET_STORE_FAST_STORE_FAST, - &&TARGET_CALL, - &&TARGET_KW_NAMES, - &&TARGET_CALL_INTRINSIC_1, - &&TARGET_CALL_INTRINSIC_2, - &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, - &&TARGET_LOAD_FROM_DICT_OR_DEREF, - &&TARGET_SET_FUNCTION_ATTRIBUTE, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_NO_KW_LEN, - &&TARGET_CALL_NO_KW_ISINSTANCE, - &&TARGET_CALL_NO_KW_LIST_APPEND, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, - &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, + &&TARGET_STORE_GLOBAL, + &&TARGET_STORE_NAME, + &&TARGET_SWAP, + &&TARGET_UNPACK_EX, + &&TARGET_UNPACK_SEQUENCE, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_YIELD_VALUE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -229,30 +230,28 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_ENTER_EXECUTOR, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, - &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, - &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_RESUME, - &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_RETURN_CONST, &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_INSTRUMENTED_FOR_ITER, + &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_JUMP_FORWARD, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, - &&TARGET_INSTRUMENTED_END_FOR, - &&TARGET_INSTRUMENTED_END_SEND, - &&TARGET_INSTRUMENTED_INSTRUCTION, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode -}; + &&_unknown_opcode}; diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index ce609bd0898741..8dbb7bfa69a926 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -22,7 +22,7 @@ verbose = False # This must be kept in sync with opcode.py -RESUME = 151 +RESUME = 166 def isprintable(b: bytes) -> bool: return all(0x20 <= c < 0x7f for c in b) @@ -297,10 +297,12 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_linetable = {co_linetable},") self.write(f"._co_cached = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") - for i, op in enumerate(code.co_code[::2]): + first_traceable = 0 + for op in code.co_code[::2]: if op == RESUME: - self.write(f"._co_firsttraceable = {i},") break + first_traceable += 1 + self.write(f"._co_firsttraceable = {first_traceable},") name_as_code = f"(PyCodeObject *)&{name}" self.finis.append(f"_PyStaticCode_Fini({name_as_code});") self.inits.append(f"_PyStaticCode_Init({name_as_code})") diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 67f4a2c2d5d76f..344709a05184ce 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -27,27 +27,6 @@ #endif /* !Py_OPCODE_H */ """ -opcode_ids_h_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_OPCODE_IDS_H -#define Py_OPCODE_IDS_H -#ifdef __cplusplus -extern "C" {{ -#endif - - -/* Instruction opcodes for compiled code */ -""".lstrip() - -opcode_ids_h_footer = """ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_IDS_H */ -""" - internal_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} @@ -83,52 +62,10 @@ def get_python_module_dict(filename): return mod def main(opcode_py, - _opcode_metadata_py='Lib/_opcode_metadata.py', - opcode_ids_h='Include/opcode_ids.h', opcode_h='Include/opcode.h', - opcode_targets_h='Python/opcode_targets.h', internal_opcode_h='Include/internal/pycore_opcode.h'): - _opcode_metadata = get_python_module_dict(_opcode_metadata_py) - opcode = get_python_module_dict(opcode_py) - opmap = opcode['opmap'] - opname = opcode['opname'] - - MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] - - NUM_OPCODES = len(opname) - used = [ False ] * len(opname) - next_op = 1 - - for name, op in opmap.items(): - used[op] = True - - specialized_opmap = {} - opname_including_specialized = opname.copy() - for name in _opcode_metadata['_specialized_instructions']: - while used[next_op]: - next_op += 1 - specialized_opmap[name] = next_op - opname_including_specialized[next_op] = name - used[next_op] = True - - with open(opcode_ids_h, 'w') as fobj: - fobj.write(opcode_ids_h_header) - - for name in opname: - if name in opmap: - op = opmap[name] - if op == MIN_INSTRUMENTED_OPCODE: - fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) - - fobj.write(DEFINE.format(name, op)) - - - for name, op in specialized_opmap.items(): - fobj.write(DEFINE.format(name, op)) - - fobj.write(opcode_ids_h_footer) with open(opcode_h, 'w') as fobj: fobj.write(opcode_h_header) @@ -143,7 +80,6 @@ def main(opcode_py, iobj.write(internal_header) iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") - iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") @@ -151,52 +87,12 @@ def main(opcode_py, iobj.write(f" [{name}] = {entries},\n") iobj.write("};\n") - deoptcodes = {} - for basic, op in opmap.items(): - if op < 256: - deoptcodes[basic] = basic - for basic, family in _opcode_metadata["_specializations"].items(): - for specialized in family: - deoptcodes[specialized] = basic - iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n") - for opt, deopt in sorted(deoptcodes.items()): - iobj.write(f" [{opt}] = {deopt},\n") - iobj.write("};\n") - iobj.write("#endif // NEED_OPCODE_TABLES\n") - - iobj.write("\n") - iobj.write(f"\nextern const char *const _PyOpcode_OpName[{NUM_OPCODES}];\n") - iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") - iobj.write(f"const char *const _PyOpcode_OpName[{NUM_OPCODES}] = {{\n") - for op, name in enumerate(opname_including_specialized): - if name[0] != "<": - op = name - iobj.write(f''' [{op}] = "{name}",\n''') - iobj.write("};\n") iobj.write("#endif // NEED_OPCODE_TABLES\n") - iobj.write("\n") - iobj.write("#define EXTRA_CASES \\\n") - for i, flag in enumerate(used): - if not flag: - iobj.write(f" case {i}: \\\n") - iobj.write(" ;\n") - iobj.write(internal_footer) - with open(opcode_targets_h, "w") as f: - targets = ["_unknown_opcode"] * 256 - for op, name in enumerate(opname_including_specialized): - if op < 256 and not name.startswith("<"): - targets[op] = f"TARGET_{name}" - - f.write("static void *opcode_targets[256] = {\n") - f.write(",\n".join([f" &&{s}" for s in targets])) - f.write("\n};\n") - print(f"{opcode_h} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], - sys.argv[5], sys.argv[6]) + main(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index e170e110f80cfb..d991cb4690063d 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -5,6 +5,7 @@ import argparse import contextlib +import itertools import os import posixpath import sys @@ -36,6 +37,12 @@ DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) +DEFAULT_OPCODE_IDS_H_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Include/opcode_ids.h") +) +DEFAULT_OPCODE_TARGETS_H_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Python/opcode_targets.h") +) DEFAULT_METADATA_OUTPUT = os.path.relpath( os.path.join(ROOT, "Include/internal/pycore_opcode_metadata.h") ) @@ -86,6 +93,20 @@ arg_parser.add_argument( "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) +arg_parser.add_argument( + "-n", + "--opcode_ids_h", + type=str, + help="Header file with opcode number definitions", + default=DEFAULT_OPCODE_IDS_H_OUTPUT, +) +arg_parser.add_argument( + "-t", + "--opcode_targets_h", + type=str, + help="File with opcode targets for computed gotos", + default=DEFAULT_OPCODE_TARGETS_H_OUTPUT, +) arg_parser.add_argument( "-m", "--metadata", @@ -225,6 +246,129 @@ def write_provenance_header(self): self.out.write_raw(self.from_source_files()) self.out.write_raw(f"{self.out.comment} Do not edit!\n") + def assign_opcode_ids(self): + """Assign IDs to opcodes""" + + ops: list[(bool, str)] = [] # (has_arg, name) for each opcode + instrumented_ops: list[str] = [] + + for instr in itertools.chain( + [instr for instr in self.instrs.values() if instr.kind != "op"], + self.macro_instrs.values()): + + name = instr.name + if name.startswith('INSTRUMENTED_'): + instrumented_ops.append(name) + else: + ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) + + # Special case: this instruction is implemented in ceval.c + # rather than bytecodes.c, so we need to add it explicitly + # here (at least until we add something to bytecodes.c to + # declare external instructions). + instrumented_ops.append('INSTRUMENTED_LINE') + + # assert lists are unique + assert len(set(ops)) == len(ops) + assert len(set(instrumented_ops)) == len(instrumented_ops) + + opname: list[str or None] = [None] * 512 + opmap: dict = {} + markers: dict = {} + + def map_op(op, name): + assert op < len(opname) + assert opname[op] is None + assert name not in opmap + opname[op] = name + opmap[name] = op + + + # 0 is reserved for cache entries. This helps debugging. + map_op(0, 'CACHE') + + # 17 is reserved as it is the initial value for the specializing counter. + # This helps catch cases where we attempt to execute a cache. + map_op(17, 'RESERVED') + + # 166 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py + map_op(166, 'RESUME') + + next_opcode = 1 + + for has_arg, name in sorted(ops): + if name in opmap: + continue # an anchored name, like CACHE + while opname[next_opcode] is not None: + next_opcode += 1 + assert next_opcode < 255 + map_op(next_opcode, name) + + if has_arg and 'HAVE_ARGUMENT' not in markers: + markers['HAVE_ARGUMENT'] = next_opcode + + # Instrumented opcodes are at the end of the valid range + min_instrumented = 254 - (len(instrumented_ops) - 1) + assert next_opcode <= min_instrumented + markers['MIN_INSTRUMENTED_OPCODE'] = min_instrumented + for i, op in enumerate(instrumented_ops): + map_op(min_instrumented + i, op) + + # Pseudo opcodes are after the valid range + for i, op in enumerate(sorted(self.pseudos)): + map_op(256 + i, op) + + assert 255 not in opmap # 255 is reserved + self.opmap = opmap + self.markers = markers + + def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): + """Write header file that defined the opcode IDs""" + + with open(opcode_ids_h_filename, "w") as f: + # Create formatter + self.out = Formatter(f, 0) + + self.write_provenance_header() + + self.out.emit("") + self.out.emit("#ifndef Py_OPCODE_IDS_H") + self.out.emit("#define Py_OPCODE_IDS_H") + self.out.emit("#ifdef __cplusplus") + self.out.emit("extern \"C\" {") + self.out.emit("#endif") + self.out.emit("") + self.out.emit("/* Instruction opcodes for compiled code */") + + def define(name, opcode): + self.out.emit(f"#define {name:<38} {opcode:>3}") + + all_pairs = [] + # the second item in the tuple sorts the markers before the ops + all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) + all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) + for i, _, name in sorted(all_pairs): + assert name is not None + define(name, i) + + self.out.emit("") + self.out.emit("#ifdef __cplusplus") + self.out.emit("}") + self.out.emit("#endif") + self.out.emit("#endif /* !Py_OPCODE_IDS_H */") + + with open(opcode_targets_filename, "w") as f: + # Create formatter + self.out = Formatter(f, 0) + + with self.out.block("static void *opcode_targets[256] =", ";"): + targets = ["_unknown_opcode"] * 256 + for name, op in self.opmap.items(): + if op < 256: + targets[op] = f"TARGET_{name}" + f.write(",\n".join([f" &&{s}" for s in targets])) + + def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: """Write instruction metadata to output file.""" @@ -378,12 +522,46 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No ): self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') + with self.metadata_item( + f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", "=", ";" + ): + for name in self.opmap: + self.out.emit(f'[{name}] = "{name}",') + + deoptcodes = {} + for name, op in self.opmap.items(): + if op < 256: + deoptcodes[name] = name + for name, family in self.families.items(): + for m in family.members: + deoptcodes[m] = name + # special case: + deoptcodes['BINARY_OP_INPLACE_ADD_UNICODE'] = 'BINARY_OP' + + with self.metadata_item( + f"const uint8_t _PyOpcode_Deopt[256]", "=", ";" + ): + for opt, deopt in sorted(deoptcodes.items()): + self.out.emit(f"[{opt}] = {deopt},") + + self.out.emit("") + self.out.emit("#define EXTRA_CASES \\") + valid_opcodes = set(self.opmap.values()) + with self.out.indent(): + for op in range(256): + if op not in valid_opcodes: + self.out.emit(f"case {op}: \\") + self.out.emit(" ;\n") + with open(pymetadata_filename, "w") as f: # Create formatter self.out = Formatter(f, 0, comment="#") self.write_provenance_header() + # emit specializations + specialized_ops = set() + self.out.emit("") self.out.emit("_specializations = {") for name, family in self.families.items(): @@ -392,6 +570,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No with self.out.indent(): for m in family.members: self.out.emit(f'"{m}",') + specialized_ops.update(family.members) self.out.emit(f"],") self.out.emit("}") @@ -402,14 +581,26 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No '_specializations["BINARY_OP"].append(' '"BINARY_OP_INPLACE_ADD_UNICODE")' ) + specialized_ops.add("BINARY_OP_INPLACE_ADD_UNICODE") - # Make list of specialized instructions + ops = sorted((id, name) for (name, id) in self.opmap.items()) + # emit specialized opmap self.out.emit("") - self.out.emit( - "_specialized_instructions = [" - "opcode for family in _specializations.values() for opcode in family" - "]" - ) + with self.out.block("_specialized_opmap ="): + for op, name in ops: + if name in specialized_ops: + self.out.emit(f"'{name}': {op},") + + # emit opmap + self.out.emit("") + with self.out.block("opmap ="): + for op, name in ops: + if name not in specialized_ops: + self.out.emit(f"'{name}': {op},") + + for name in ['MIN_INSTRUMENTED_OPCODE', 'HAVE_ARGUMENT']: + self.out.emit(f"{name} = {self.markers[name]}") + def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" @@ -683,6 +874,9 @@ def main(): # These raise OSError if output can't be written a.write_instructions(args.output, args.emit_line_directives) + + a.assign_opcode_ids() + a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index f798b2f772d08a..2d198506fb5c6c 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -17,7 +17,7 @@ DEFAULT_DIR = "/tmp/py_stats/" #Create list of all instruction names -specialized = iter(opcode._specialized_instructions) +specialized = iter(opcode._specialized_opmap.keys()) opname = ["<0>"] for name in opcode.opname[1:]: if name.startswith("<"): @@ -244,7 +244,7 @@ def categorized_counts(opcode_stats): specialized = 0 not_specialized = 0 specialized_instructions = { - op for op in opcode._specialized_instructions + op for op in opcode._specialized_opmap.keys() if "__" not in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: From dc8fdf5fd5f572e87152883f15f35c354fa4b4bf Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 16 Aug 2023 16:26:43 -0700 Subject: [PATCH 169/598] gh-106581: Split `CALL_PY_EXACT_ARGS` into uops (#107760) * Split `CALL_PY_EXACT_ARGS` into uops This is only the first step for doing `CALL` in Tier 2. The next step involves tracing into the called code object and back. After that we'll have to do the remaining `CALL` specialization. Finally we'll have to deal with `KW_NAMES`. Note: this moves setting `frame->return_offset` directly in front of `DISPATCH_INLINED()`, to make it easier to move it into `_PUSH_FRAME`. --- Include/internal/pycore_opcode_metadata.h | 47 +++++++++- Lib/test/test_capi/test_misc.py | 17 ++++ Python/abstract_interp_cases.c.h | 28 ++++++ Python/bytecodes.c | 82 ++++++++++++++---- Python/ceval.c | 6 +- Python/ceval_macros.h | 5 ++ Python/executor.c | 2 + Python/executor_cases.c.h | 98 +++++++++++++++++++-- Python/generated_cases.c.h | 101 ++++++++++++++++------ Python/optimizer.c | 9 ++ Tools/cases_generator/flags.py | 2 +- Tools/cases_generator/generate_cases.py | 34 ++++++-- Tools/cases_generator/instructions.py | 37 +++----- Tools/cases_generator/stacking.py | 60 ++++++++----- 14 files changed, 412 insertions(+), 116 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7dbd78b32a8957..afe8aa172b703b 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -52,10 +52,16 @@ #define _ITER_CHECK_RANGE 328 #define _IS_ITER_EXHAUSTED_RANGE 329 #define _ITER_NEXT_RANGE 330 -#define _POP_JUMP_IF_FALSE 331 -#define _POP_JUMP_IF_TRUE 332 -#define JUMP_TO_TOP 333 -#define INSERT 334 +#define _CHECK_PEP_523 331 +#define _CHECK_FUNCTION_EXACT_ARGS 332 +#define _CHECK_STACK_SPACE 333 +#define _INIT_CALL_PY_EXACT_ARGS 334 +#define _PUSH_FRAME 335 +#define _POP_JUMP_IF_FALSE 336 +#define _POP_JUMP_IF_TRUE 337 +#define JUMP_TO_TOP 338 +#define SAVE_CURRENT_IP 339 +#define INSERT 340 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -473,6 +479,16 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg + 2; case CALL_BOUND_METHOD_EXACT_ARGS: return oparg + 2; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_STACK_SPACE: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return oparg + 2; + case _PUSH_FRAME: + return 1; case CALL_PY_EXACT_ARGS: return oparg + 2; case CALL_PY_WITH_DEFAULTS: @@ -561,6 +577,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case SAVE_IP: return 0; + case SAVE_CURRENT_IP: + return 0; case EXIT_TRACE: return 0; case INSERT: @@ -987,6 +1005,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case CALL_BOUND_METHOD_EXACT_ARGS: return 1; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_STACK_SPACE: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return 1; + case _PUSH_FRAME: + return 1; case CALL_PY_EXACT_ARGS: return 1; case CALL_PY_WITH_DEFAULTS: @@ -1075,6 +1103,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case SAVE_IP: return 0; + case SAVE_CURRENT_IP: + return 0; case EXIT_TRACE: return 0; case INSERT: @@ -1088,6 +1118,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, + INSTR_FMT_IBC0, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, @@ -1132,6 +1163,7 @@ struct opcode_macro_expansion { #define OPARG_CACHE_4 4 #define OPARG_TOP 5 #define OPARG_BOTTOM 6 +#define OPARG_SAVE_IP 7 #define OPCODE_METADATA_FMT(OP) (_PyOpcode_opcode_metadata[(OP)].instr_format) #define SAME_OPCODE_METADATA(OP1, OP2) \ @@ -1474,6 +1506,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { SAVE_IP, 7, 3 }, { SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_NO_KW_TYPE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TYPE_1, 0, 0 } } }, [CALL_NO_KW_STR_1] = { .nuops = 1, .uops = { { CALL_NO_KW_STR_1, 0, 0 } } }, [CALL_NO_KW_TUPLE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TUPLE_1, 0, 0 } } }, @@ -1531,9 +1564,15 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_PUSH_FRAME] = "_PUSH_FRAME", [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [JUMP_TO_TOP] = "JUMP_TO_TOP", + [SAVE_CURRENT_IP] = "SAVE_CURRENT_IP", [INSERT] = "INSERT", }; #endif // NEED_OPCODE_METADATA diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c81212202d9ef2..3dfbfdc26e7416 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2618,6 +2618,23 @@ def testfunc(it): with self.assertRaises(StopIteration): next(it) + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(10) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_PUSH_FRAME", uops) + + if __name__ == "__main__": unittest.main() diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 6bfcf534646b1e..eef071119bcd84 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -612,6 +612,30 @@ break; } + case _CHECK_PEP_523: { + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + break; + } + + case _CHECK_STACK_SPACE: { + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _PUSH_FRAME: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + case CALL_NO_KW_TYPE_1: { STACK_SHRINK(oparg); STACK_SHRINK(1); @@ -751,6 +775,10 @@ break; } + case SAVE_CURRENT_IP: { + break; + } + case EXIT_TRACE: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2a5ad2c942fb3e..ae2923c65b3308 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -956,13 +956,13 @@ dummy_func( { PyGenObject *gen = (PyGenObject *)receiver; _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -995,13 +995,13 @@ dummy_func( DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } @@ -2587,7 +2587,6 @@ dummy_func( DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; _PyFrame_StackPush(gen_frame, Py_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; @@ -2595,6 +2594,7 @@ dummy_func( SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } @@ -2949,32 +2949,72 @@ dummy_func( GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - ASSERT_KWNAMES_IS_NULL(); + op(_CHECK_PEP_523, (--)) { DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != argcount, CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = args[i]; } - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + } + + // The 'unused' output effect represents the return value + // (which will be pushed when the frame returns). + // It is needed so CALL_PY_EXACT_ARGS matches its family. + op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. frame->return_offset = 0; - DISPATCH_INLINED(new_frame); + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + #if TIER_ONE + frame = cframe.current_frame = new_frame; + goto start_frame; + #endif + #if TIER_TWO + frame = tstate->cframe->current_frame = new_frame; + ERROR_IF(_Py_EnterRecursivePy(tstate), exit_unwind); + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif } + macro(CALL_PY_EXACT_ARGS) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _INIT_CALL_PY_EXACT_ARGS + + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _PUSH_FRAME; + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3735,6 +3775,16 @@ dummy_func( frame->prev_instr = ip_offset + oparg; } + op(SAVE_CURRENT_IP, (--)) { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + op(EXIT_TRACE, (--)) { frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/ceval.c b/Python/ceval.c index b966399a342d08..26e741ed7c7547 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -602,11 +602,6 @@ int _Py_CheckRecursiveCallPy( return 0; } -static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { - return (tstate->py_recursion_remaining-- <= 0) && - _Py_CheckRecursiveCallPy(tstate); -} - static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; @@ -770,6 +765,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif { +#define TIER_ONE 1 #include "generated_cases.c.h" /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8dc8b754485856..5e2db1e0b394e6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -364,3 +364,8 @@ static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { #else #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif + +static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { + return (tstate->py_recursion_remaining-- <= 0) && + _Py_CheckRecursiveCallPy(tstate); +} diff --git a/Python/executor.c b/Python/executor.c index 4a18618c0c6c0c..5a571e6da4673f 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -81,6 +81,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject OBJECT_STAT_INC(optimization_uops_executed); switch (opcode) { +#define TIER_TWO 2 #include "executor_cases.c.h" default: @@ -106,6 +107,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject pop_2_error: STACK_SHRINK(1); pop_1_error: +pop_1_exit_unwind: STACK_SHRINK(1); error: // On ERROR_IF we return NULL as the frame. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 85d27777423abd..b3dd3133530562 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -103,7 +103,6 @@ } case TO_BOOL: { - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -363,7 +362,6 @@ } case BINARY_SUBSCR: { - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *res; @@ -557,7 +555,6 @@ } case STORE_SUBSCR: { - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *v; @@ -862,7 +859,6 @@ } case UNPACK_SEQUENCE: { - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION @@ -950,7 +946,6 @@ } case STORE_ATTR: { - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; owner = stack_pointer[-1]; @@ -1061,7 +1056,6 @@ } case LOAD_GLOBAL: { - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION @@ -1554,7 +1548,6 @@ } case LOAD_ATTR: { - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -1650,7 +1643,6 @@ } case COMPARE_OP: { - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2155,6 +2147,84 @@ break; } + case _CHECK_PEP_523: { + DEOPT_IF(tstate->interp->eval_frame, CALL); + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + PyObject *self_or_null; + PyObject *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)operand; + ASSERT_KWNAMES_IS_NULL(); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + break; + } + + case _CHECK_STACK_SPACE: { + PyObject *callable; + callable = stack_pointer[-2 - oparg]; + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = (PyObject *)new_frame; + break; + } + + case _PUSH_FRAME: { + _PyInterpreterFrame *new_frame; + new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; + STACK_SHRINK(1); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + frame->return_offset = 0; + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + #if TIER_ONE + frame = cframe.current_frame = new_frame; + goto start_frame; + #endif + #if TIER_TWO + frame = tstate->cframe->current_frame = new_frame; + if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + break; + } + case CALL_NO_KW_TYPE_1: { PyObject **args; PyObject *null; @@ -2656,7 +2726,6 @@ } case BINARY_OP: { - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; PyObject *res; @@ -2726,6 +2795,17 @@ break; } + case SAVE_CURRENT_IP: { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + break; + } + case EXIT_TRACE: { frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2661a39e047c4d..11d560a6e77adf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1191,13 +1191,13 @@ { PyGenObject *gen = (PyGenObject *)receiver; _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -1237,13 +1237,13 @@ DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } @@ -3343,7 +3343,6 @@ DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; _PyFrame_StackPush(gen_frame, Py_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; @@ -3351,6 +3350,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); STACK_GROW(1); } @@ -3764,38 +3764,83 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args; PyObject *self_or_null; PyObject *callable; - args = stack_pointer - oparg; + PyObject **args; + _PyInterpreterFrame *new_frame; + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&next_instr[1].cache); - ASSERT_KWNAMES_IS_NULL(); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + { + uint32_t func_version = read_u32(&next_instr[1].cache); + ASSERT_KWNAMES_IS_NULL(); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + callable = stack_pointer[-2 - oparg]; + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != argcount, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + // _INIT_CALL_PY_EXACT_ARGS + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } } - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); + // SAVE_CURRENT_IP + next_instr += 3; + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _PUSH_FRAME STACK_SHRINK(oparg); - STACK_SHRINK(1); + STACK_SHRINK(2); + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + frame->return_offset = 0; + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + #if TIER_ONE + frame = cframe.current_frame = new_frame; + goto start_frame; + #endif + #if TIER_TWO + frame = tstate->cframe->current_frame = new_frame; + if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(CALL_PY_WITH_DEFAULTS) { diff --git a/Python/optimizer.c b/Python/optimizer.c index d3ac2424038ef9..559c4ae987263e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -606,6 +606,10 @@ translate_bytecode_to_trace( case OPARG_BOTTOM: // Second half of super-instr oparg = orig_oparg & 0xF; break; + case OPARG_SAVE_IP: // op==SAVE_IP; oparg=next instr + oparg = INSTR_IP(instr + offset, code); + break; + default: fprintf(stderr, "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", @@ -615,6 +619,11 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } ADD_TO_TRACE(expansion->uops[i].uop, oparg, operand); + if (expansion->uops[i].uop == _PUSH_FRAME) { + assert(i + 1 == nuops); + ADD_TO_TRACE(SAVE_IP, 0, 0); + goto done; + } } break; } diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index f7ebdeb0d65677..962f003b194dbd 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -92,7 +92,7 @@ def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: if text == "#if": if ( i + 1 < len(node.tokens) - and node.tokens[i + 1].text == "ENABLE_SPECIALIZATION" + and node.tokens[i + 1].text in ("ENABLE_SPECIALIZATION", "TIER_ONE") ): skipping = True elif text in ("#else", "#endif"): diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index d991cb4690063d..f31b6658d6599e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -25,6 +25,7 @@ PseudoInstruction, StackEffect, OverriddenInstructionPlaceHolder, + TIER_ONE, TIER_TWO, ) import parsing @@ -65,6 +66,7 @@ "OPARG_CACHE_4": 4, "OPARG_TOP": 5, "OPARG_BOTTOM": 6, + "OPARG_SAVE_IP": 7, } INSTR_FMT_PREFIX = "INSTR_FMT_" @@ -501,7 +503,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No if instr.kind == "inst" and instr.is_viable_uop(): # Construct a dummy Component -- input/output mappings are not used part = Component(instr, instr.active_caches) - self.write_macro_expansions(instr.name, [part]) + self.write_macro_expansions( + instr.name, [part], instr.cache_offset + ) elif instr.kind == "inst" and variable_used( instr.inst, "oparg1" ): @@ -511,7 +515,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_super_expansions(instr.name) case parsing.Macro(): mac = self.macro_instrs[thing.name] - self.write_macro_expansions(mac.name, mac.parts) + self.write_macro_expansions( + mac.name, mac.parts, mac.cache_offset + ) case parsing.Pseudo(): pass case _: @@ -630,7 +636,9 @@ def add(name: str) -> None: if instr.kind == "op" and instr.is_viable_uop(): add(instr.name) - def write_macro_expansions(self, name: str, parts: MacroParts) -> None: + def write_macro_expansions( + self, name: str, parts: MacroParts, cache_offset: int + ) -> None: """Write the macro expansions for a macro-instruction.""" # TODO: Refactor to share code with write_cody(), is_viaible_uop(), etc. offset = 0 # Cache effect offset @@ -650,7 +658,10 @@ def write_macro_expansions(self, name: str, parts: MacroParts) -> None: ) return if not part.active_caches: - size, offset = OPARG_SIZES["OPARG_FULL"], 0 + if part.instr.name == "SAVE_IP": + size, offset = OPARG_SIZES["OPARG_SAVE_IP"], cache_offset + else: + size, offset = OPARG_SIZES["OPARG_FULL"], 0 else: # If this assert triggers, is_viable_uops() lied assert len(part.active_caches) == 1, (name, part.instr.name) @@ -753,7 +764,9 @@ def write_instructions( case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - stacking.write_macro_instr(mac, self.out, self.families.get(mac.name)) + stacking.write_macro_instr( + mac, self.out, self.families.get(mac.name) + ) # self.write_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): pass @@ -789,7 +802,9 @@ def write_executor_instructions( n_instrs += 1 self.out.emit("") with self.out.block(f"case {thing.name}:"): - instr.write(self.out, tier=TIER_TWO) + stacking.write_single_instr( + instr, self.out, tier=TIER_TWO + ) if instr.check_eval_breaker: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit("break;") @@ -851,8 +866,13 @@ def write_instr(self, instr: Instruction) -> None: with self.out.block(f"TARGET({name})"): if instr.predicted: self.out.emit(f"PREDICTED({name});") - instr.write(self.out) + self.out.static_assert_family_size( + instr.name, instr.family, instr.cache_offset + ) + stacking.write_single_instr(instr, self.out, tier=TIER_ONE) if not instr.always_exits: + if instr.cache_offset: + self.out.emit(f"next_instr += {instr.cache_offset};") if instr.check_eval_breaker: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit(f"DISPATCH();") diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index a505df08fa265b..9143ae0db7be81 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -59,7 +59,7 @@ class Instruction: block_line: int # First line of block in original code # Computed by constructor - always_exits: bool + always_exits: str # If the block always exits, its last line; else "" has_deopt: bool cache_offset: int cache_effects: list[parsing.CacheEffect] @@ -120,13 +120,13 @@ def __init__(self, inst: parsing.InstDef): def is_viable_uop(self) -> bool: """Whether this instruction is viable as a uop.""" dprint: typing.Callable[..., None] = lambda *args, **kwargs: None - # if self.name.startswith("CALL"): - # dprint = print + if "FRAME" in self.name: + dprint = print if self.name == "EXIT_TRACE": return True # This has 'return frame' but it's okay if self.always_exits: - dprint(f"Skipping {self.name} because it always exits") + dprint(f"Skipping {self.name} because it always exits: {self.always_exits}") return False if len(self.active_caches) > 1: # print(f"Skipping {self.name} because it has >1 cache entries") @@ -140,23 +140,6 @@ def is_viable_uop(self) -> bool: res = False return res - def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: - """Write one instruction, sans prologue and epilogue.""" - - # Write a static assertion that a family's cache size is correct - out.static_assert_family_size(self.name, self.family, self.cache_offset) - - # Write input stack effect variable declarations and initializations - stacking.write_single_instr(self, out, tier) - - # Skip the rest if the block always exits - if self.always_exits: - return - - # Write cache effect - if tier == TIER_ONE and self.cache_offset: - out.emit(f"next_instr += {self.cache_offset};") - def write_body( self, out: Formatter, @@ -341,16 +324,16 @@ def extract_block_text(block: parsing.Block) -> tuple[list[str], bool, int]: return blocklines, check_eval_breaker, block_line -def always_exits(lines: list[str]) -> bool: +def always_exits(lines: list[str]) -> str: """Determine whether a block always ends in a return/goto/etc.""" if not lines: - return False + return "" line = lines[-1].rstrip() # Indent must match exactly (TODO: Do something better) if line[:12] != " " * 12: - return False + return "" line = line[12:] - return line.startswith( + if line.startswith( ( "goto ", "return ", @@ -359,4 +342,6 @@ def always_exits(lines: list[str]) -> bool: "Py_UNREACHABLE()", "ERROR_IF(true, ", ) - ) + ): + return line + return "" diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 31a21e026cb49c..8361eb99f88a7c 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -1,6 +1,7 @@ import dataclasses import typing +from flags import variable_used_unspecialized from formatting import ( Formatter, UNUSED, @@ -146,6 +147,8 @@ class EffectManager: # Track offsets from stack pointer min_offset: StackOffset final_offset: StackOffset + # Link to previous manager + pred: "EffectManager | None" = None def __init__( self, @@ -167,7 +170,8 @@ def __init__( self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) self.final_offset.higher(eff) - if pred: + self.pred = pred + while pred: # Replace push(x) + pop(y) with copy(x, y). # Check that the sources and destinations are disjoint. sources: set[str] = set() @@ -192,6 +196,11 @@ def __init__( sources, destinations, ) + # See if we can get more copies of a earlier predecessor. + if self.peeks and not pred.pokes and not pred.peeks: + pred = pred.pred + else: + pred = None # Break def adjust_deeper(self, eff: StackEffect) -> None: for peek in self.peeks: @@ -295,6 +304,7 @@ def write_single_instr( [Component(instr, instr.active_caches)], out, tier, + 0, ) except AssertionError as err: raise AssertionError(f"Error writing instruction {instr.name}") from err @@ -303,37 +313,32 @@ def write_single_instr( def write_macro_instr( mac: MacroInstruction, out: Formatter, family: Family | None ) -> None: - parts = [part for part in mac.parts if isinstance(part, Component)] - - cache_adjust = 0 - for part in mac.parts: - match part: - case CacheEffect(size=size): - cache_adjust += size - case Component(instr=instr): - cache_adjust += instr.cache_offset - case _: - typing.assert_never(part) - + parts = [ + part + for part in mac.parts + if isinstance(part, Component) and part.instr.name != "SAVE_IP" + ] out.emit("") with out.block(f"TARGET({mac.name})"): if mac.predicted: out.emit(f"PREDICTED({mac.name});") - out.static_assert_family_size(mac.name, family, cache_adjust) + out.static_assert_family_size(mac.name, family, mac.cache_offset) try: - write_components(parts, out, TIER_ONE) + next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err - if cache_adjust: - out.emit(f"next_instr += {cache_adjust};") - out.emit("DISPATCH();") + if not parts[-1].instr.always_exits and not next_instr_is_set: + if mac.cache_offset: + out.emit(f"next_instr += {mac.cache_offset};") + out.emit("DISPATCH();") def write_components( parts: list[Component], out: Formatter, tier: Tiers, -) -> None: + cache_offset: int, +) -> bool: managers = get_managers(parts) all_vars: dict[str, StackEffect] = {} @@ -354,6 +359,7 @@ def write_components( for name, eff in all_vars.items(): out.declare(eff, None) + next_instr_is_set = False for mgr in managers: if len(parts) > 1: out.emit(f"// {mgr.instr.name}") @@ -374,13 +380,25 @@ def write_components( poke.as_stack_effect(lax=True), ) + if mgr.instr.name == "_PUSH_FRAME": + # Adjust stack to min_offset (input effects materialized) + out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + + if mgr.instr.name == "SAVE_CURRENT_IP": + next_instr_is_set = True + if cache_offset: + out.emit(f"next_instr += {cache_offset};") + if len(parts) == 1: mgr.instr.write_body(out, 0, mgr.active_caches, tier) else: with out.block(""): mgr.instr.write_body(out, -4, mgr.active_caches, tier) - if mgr is managers[-1]: + if mgr is managers[-1] and not next_instr_is_set: + # TODO: Explain why this adjustment is needed. out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) # Use clone() since adjust_inverse() mutates final_offset mgr.adjust_inverse(mgr.final_offset.clone()) @@ -392,6 +410,8 @@ def write_components( poke.effect, ) + return next_instr_is_set + def write_single_instr_for_abstract_interp( instr: Instruction, out: Formatter From c9d83f93d804b80ee14480466ebee63a6f97dac2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 17 Aug 2023 09:44:05 +0300 Subject: [PATCH 170/598] gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) --- Doc/conf.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/Doc/conf.py b/Doc/conf.py index 19e05e1aa8fe19..19eab1a9340e47 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -157,6 +157,77 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), +] + +# Temporary undocumented names. +# In future this list must be empty. +nitpick_ignore += [ + # C API: Standard Python exception classes + ('c:data', 'PyExc_ArithmeticError'), + ('c:data', 'PyExc_AssertionError'), + ('c:data', 'PyExc_AttributeError'), + ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BlockingIOError'), + ('c:data', 'PyExc_BrokenPipeError'), + ('c:data', 'PyExc_BufferError'), + ('c:data', 'PyExc_ChildProcessError'), + ('c:data', 'PyExc_ConnectionAbortedError'), + ('c:data', 'PyExc_ConnectionError'), + ('c:data', 'PyExc_ConnectionRefusedError'), + ('c:data', 'PyExc_ConnectionResetError'), + ('c:data', 'PyExc_EOFError'), + ('c:data', 'PyExc_Exception'), + ('c:data', 'PyExc_FileExistsError'), + ('c:data', 'PyExc_FileNotFoundError'), + ('c:data', 'PyExc_FloatingPointError'), + ('c:data', 'PyExc_GeneratorExit'), + ('c:data', 'PyExc_ImportError'), + ('c:data', 'PyExc_IndentationError'), + ('c:data', 'PyExc_IndexError'), + ('c:data', 'PyExc_InterruptedError'), + ('c:data', 'PyExc_IsADirectoryError'), + ('c:data', 'PyExc_KeyboardInterrupt'), + ('c:data', 'PyExc_KeyError'), + ('c:data', 'PyExc_LookupError'), + ('c:data', 'PyExc_MemoryError'), + ('c:data', 'PyExc_ModuleNotFoundError'), + ('c:data', 'PyExc_NameError'), + ('c:data', 'PyExc_NotADirectoryError'), + ('c:data', 'PyExc_NotImplementedError'), + ('c:data', 'PyExc_OSError'), + ('c:data', 'PyExc_OverflowError'), + ('c:data', 'PyExc_PermissionError'), + ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_RecursionError'), + ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RuntimeError'), + ('c:data', 'PyExc_StopAsyncIteration'), + ('c:data', 'PyExc_StopIteration'), + ('c:data', 'PyExc_SyntaxError'), + ('c:data', 'PyExc_SystemError'), + ('c:data', 'PyExc_SystemExit'), + ('c:data', 'PyExc_TabError'), + ('c:data', 'PyExc_TimeoutError'), + ('c:data', 'PyExc_TypeError'), + ('c:data', 'PyExc_UnboundLocalError'), + ('c:data', 'PyExc_UnicodeDecodeError'), + ('c:data', 'PyExc_UnicodeEncodeError'), + ('c:data', 'PyExc_UnicodeError'), + ('c:data', 'PyExc_UnicodeTranslateError'), + ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_ZeroDivisionError'), + # C API: Standard Python warning classes + ('c:data', 'PyExc_BytesWarning'), + ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_FutureWarning'), + ('c:data', 'PyExc_ImportWarning'), + ('c:data', 'PyExc_PendingDeprecationWarning'), + ('c:data', 'PyExc_ResourceWarning'), + ('c:data', 'PyExc_RuntimeWarning'), + ('c:data', 'PyExc_SyntaxWarning'), + ('c:data', 'PyExc_UnicodeWarning'), + ('c:data', 'PyExc_UserWarning'), + ('c:data', 'PyExc_Warning'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. From 1344cfac43a1920c596b0e8718ca0567889e697b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 17 Aug 2023 08:45:48 +0200 Subject: [PATCH 171/598] gh-105539: Explict resource management for connection objects in sqlite3 tests (#108017) - Use memory_database() helper - Move test utility functions to util.py - Add convenience memory database mixin - Add check() helper for closed connection tests --- Lib/test/test_sqlite3/test_backup.py | 52 ++-- Lib/test/test_sqlite3/test_dbapi.py | 253 +++++++++----------- Lib/test/test_sqlite3/test_dump.py | 10 +- Lib/test/test_sqlite3/test_factory.py | 58 ++--- Lib/test/test_sqlite3/test_hooks.py | 79 +++--- Lib/test/test_sqlite3/test_regression.py | 123 +++++----- Lib/test/test_sqlite3/test_transactions.py | 33 +-- Lib/test/test_sqlite3/test_userfunctions.py | 74 ++---- Lib/test/test_sqlite3/util.py | 78 ++++++ 9 files changed, 373 insertions(+), 387 deletions(-) create mode 100644 Lib/test/test_sqlite3/util.py diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index 87ab29c54d65e2..4584d976bce0c6 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -1,6 +1,8 @@ import sqlite3 as sqlite import unittest +from .util import memory_database + class BackupTests(unittest.TestCase): def setUp(self): @@ -32,32 +34,32 @@ def test_bad_target_same_connection(self): self.cx.backup(self.cx) def test_bad_target_closed_connection(self): - bck = sqlite.connect(':memory:') - bck.close() - with self.assertRaises(sqlite.ProgrammingError): - self.cx.backup(bck) + with memory_database() as bck: + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) def test_bad_source_closed_connection(self): - bck = sqlite.connect(':memory:') - source = sqlite.connect(":memory:") - source.close() - with self.assertRaises(sqlite.ProgrammingError): - source.backup(bck) + with memory_database() as bck: + source = sqlite.connect(":memory:") + source.close() + with self.assertRaises(sqlite.ProgrammingError): + source.backup(bck) def test_bad_target_in_transaction(self): - bck = sqlite.connect(':memory:') - bck.execute('CREATE TABLE bar (key INTEGER)') - bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) - with self.assertRaises(sqlite.OperationalError) as cm: - self.cx.backup(bck) + with memory_database() as bck: + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) def test_keyword_only_args(self): with self.assertRaises(TypeError): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, 1) def test_simple(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck) self.verify_backup(bck) @@ -67,7 +69,7 @@ def test_progress(self): def progress(status, remaining, total): journal.append(status) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -81,7 +83,7 @@ def test_progress_all_pages_at_once_1(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.verify_backup(bck) @@ -94,7 +96,7 @@ def test_progress_all_pages_at_once_2(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=-1, progress=progress) self.verify_backup(bck) @@ -103,7 +105,7 @@ def progress(status, remaining, total): def test_non_callable_progress(self): with self.assertRaises(TypeError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress='bar') self.assertEqual(str(cm.exception), 'progress argument must be a callable') @@ -116,7 +118,7 @@ def progress(status, remaining, total): self.cx.commit() journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -140,12 +142,12 @@ def progress(status, remaining, total): self.assertEqual(str(err.exception), 'nearly out of space') def test_database_source_name(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='main') - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='temp') with self.assertRaises(sqlite.OperationalError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='non-existing') self.assertIn("unknown database", str(cm.exception)) @@ -153,7 +155,7 @@ def test_database_source_name(self): self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) self.cx.commit() - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='attached_db') self.verify_backup(bck) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index c9a9e1353938c6..df3c2ea8d1dbda 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -33,26 +33,13 @@ SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, is_emscripten, is_wasi ) +from test.support import gc_collect from test.support import threading_helper from _testcapi import INT_MAX, ULLONG_MAX from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath - -# Helper for temporary memory databases -def memory_database(*args, **kwargs): - cx = sqlite.connect(":memory:", *args, **kwargs) - return contextlib.closing(cx) - - -# Temporarily limit a database connection parameter -@contextlib.contextmanager -def cx_limit(cx, category=sqlite.SQLITE_LIMIT_SQL_LENGTH, limit=128): - try: - _prev = cx.setlimit(category, limit) - yield limit - finally: - cx.setlimit(category, _prev) +from .util import memory_database, cx_limit class ModuleTests(unittest.TestCase): @@ -326,9 +313,9 @@ def test_extended_error_code_on_exception(self): self.assertEqual(exc.sqlite_errorname, "SQLITE_CONSTRAINT_CHECK") def test_disallow_instantiation(self): - cx = sqlite.connect(":memory:") - check_disallow_instantiation(self, type(cx("select 1"))) - check_disallow_instantiation(self, sqlite.Blob) + with memory_database() as cx: + check_disallow_instantiation(self, type(cx("select 1"))) + check_disallow_instantiation(self, sqlite.Blob) def test_complete_statement(self): self.assertFalse(sqlite.complete_statement("select t")) @@ -342,6 +329,7 @@ def setUp(self): cu = self.cx.cursor() cu.execute("create table test(id integer primary key, name text)") cu.execute("insert into test(name) values (?)", ("foo",)) + cu.close() def tearDown(self): self.cx.close() @@ -412,21 +400,22 @@ def test_exceptions(self): def test_in_transaction(self): # Can't use db from setUp because we want to test initial state. - cx = sqlite.connect(":memory:") - cu = cx.cursor() - self.assertEqual(cx.in_transaction, False) - cu.execute("create table transactiontest(id integer primary key, name text)") - self.assertEqual(cx.in_transaction, False) - cu.execute("insert into transactiontest(name) values (?)", ("foo",)) - self.assertEqual(cx.in_transaction, True) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, True) - cx.commit() - self.assertEqual(cx.in_transaction, False) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, False) + with memory_database() as cx: + cu = cx.cursor() + self.assertEqual(cx.in_transaction, False) + cu.execute("create table transactiontest(id integer primary key, name text)") + self.assertEqual(cx.in_transaction, False) + cu.execute("insert into transactiontest(name) values (?)", ("foo",)) + self.assertEqual(cx.in_transaction, True) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, True) + cx.commit() + self.assertEqual(cx.in_transaction, False) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, False) + cu.close() def test_in_transaction_ro(self): with self.assertRaises(AttributeError): @@ -450,10 +439,9 @@ def test_connection_exceptions(self): self.assertIs(getattr(sqlite, exc), getattr(self.cx, exc)) def test_interrupt_on_closed_db(self): - cx = sqlite.connect(":memory:") - cx.close() + self.cx.close() with self.assertRaises(sqlite.ProgrammingError): - cx.interrupt() + self.cx.interrupt() def test_interrupt(self): self.assertIsNone(self.cx.interrupt()) @@ -521,29 +509,29 @@ def test_connection_init_good_isolation_levels(self): self.assertEqual(cx.isolation_level, level) def test_connection_reinit(self): - db = ":memory:" - cx = sqlite.connect(db) - cx.text_factory = bytes - cx.row_factory = sqlite.Row - cu = cx.cursor() - cu.execute("create table foo (bar)") - cu.executemany("insert into foo (bar) values (?)", - ((str(v),) for v in range(4))) - cu.execute("select bar from foo") - - rows = [r for r in cu.fetchmany(2)] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], [b"0", b"1"]) - - cx.__init__(db) - cx.execute("create table foo (bar)") - cx.executemany("insert into foo (bar) values (?)", - ((v,) for v in ("a", "b", "c", "d"))) - - # This uses the old database, old row factory, but new text factory - rows = [r for r in cu.fetchall()] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], ["2", "3"]) + with memory_database() as cx: + cx.text_factory = bytes + cx.row_factory = sqlite.Row + cu = cx.cursor() + cu.execute("create table foo (bar)") + cu.executemany("insert into foo (bar) values (?)", + ((str(v),) for v in range(4))) + cu.execute("select bar from foo") + + rows = [r for r in cu.fetchmany(2)] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], [b"0", b"1"]) + + cx.__init__(":memory:") + cx.execute("create table foo (bar)") + cx.executemany("insert into foo (bar) values (?)", + ((v,) for v in ("a", "b", "c", "d"))) + + # This uses the old database, old row factory, but new text factory + rows = [r for r in cu.fetchall()] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], ["2", "3"]) + cu.close() def test_connection_bad_reinit(self): cx = sqlite.connect(":memory:") @@ -591,11 +579,11 @@ def test_connect_positional_arguments(self): "parameters in Python 3.15." ) with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - sqlite.connect(":memory:", 1.0) + cx = sqlite.connect(":memory:", 1.0) + cx.close() self.assertEqual(cm.filename, __file__) - class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): self.cx = sqlite.Connection.__new__(sqlite.Connection) @@ -1571,12 +1559,12 @@ def run(con, err): except sqlite.Error: err.append("multi-threading not allowed") - con = sqlite.connect(":memory:", check_same_thread=False) - err = [] - t = threading.Thread(target=run, kwargs={"con": con, "err": err}) - t.start() - t.join() - self.assertEqual(len(err), 0, "\n".join(err)) + with memory_database(check_same_thread=False) as con: + err = [] + t = threading.Thread(target=run, kwargs={"con": con, "err": err}) + t.start() + t.join() + self.assertEqual(len(err), 0, "\n".join(err)) class ConstructorTests(unittest.TestCase): @@ -1602,9 +1590,16 @@ def test_binary(self): b = sqlite.Binary(b"\0'") class ExtensionTests(unittest.TestCase): + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + def test_script_string_sql(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cur cur.executescript(""" -- bla bla /* a stupid comment */ @@ -1616,40 +1611,40 @@ def test_script_string_sql(self): self.assertEqual(res, 5) def test_script_syntax_error(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(x); asdf; create table test2(x)") + self.cur.executescript(""" + CREATE TABLE test(x); + asdf; + CREATE TABLE test2(x) + """) def test_script_error_normal(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(sadfsadfdsa); select foo from hurz;") + self.cur.executescript(""" + CREATE TABLE test(sadfsadfdsa); + SELECT foo FROM hurz; + """) def test_cursor_executescript_as_bytes(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(TypeError): - cur.executescript(b"create table test(foo); insert into test(foo) values (5);") + self.cur.executescript(b""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) def test_cursor_executescript_with_null_characters(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(ValueError): - cur.executescript(""" - create table a(i);\0 - insert into a(i) values (5); - """) + self.cur.executescript(""" + CREATE TABLE a(i);\0 + INSERT INTO a(i) VALUES (5); + """) def test_cursor_executescript_with_surrogates(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(UnicodeEncodeError): - cur.executescript(""" - create table a(s); - insert into a(s) values ('\ud8ff'); - """) + self.cur.executescript(""" + CREATE TABLE a(s); + INSERT INTO a(s) VALUES ('\ud8ff'); + """) def test_cursor_executescript_too_large_script(self): msg = "query string is too large" @@ -1659,19 +1654,18 @@ def test_cursor_executescript_too_large_script(self): cx.executescript("select 'too large'".ljust(lim+1)) def test_cursor_executescript_tx_control(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("begin") self.assertTrue(con.in_transaction) con.executescript("select 1") self.assertFalse(con.in_transaction) def test_connection_execute(self): - con = sqlite.connect(":memory:") - result = con.execute("select 5").fetchone()[0] + result = self.con.execute("select 5").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.execute") def test_connection_executemany(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table test(foo)") con.executemany("insert into test(foo) values (?)", [(3,), (4,)]) result = con.execute("select foo from test order by foo").fetchall() @@ -1679,47 +1673,44 @@ def test_connection_executemany(self): self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany") def test_connection_executescript(self): - con = sqlite.connect(":memory:") - con.executescript("create table test(foo); insert into test(foo) values (5);") + con = self.con + con.executescript(""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) result = con.execute("select foo from test").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.executescript") + class ClosedConTests(unittest.TestCase): + def check(self, fn, *args, **kwds): + regex = "Cannot operate on a closed database." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + fn(*args, **kwds) + + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + self.con.close() + def test_closed_con_cursor(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur = con.cursor() + self.check(self.con.cursor) def test_closed_con_commit(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.commit() + self.check(self.con.commit) def test_closed_con_rollback(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.rollback() + self.check(self.con.rollback) def test_closed_cur_execute(self): - con = sqlite.connect(":memory:") - cur = con.cursor() - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur.execute("select 4") + self.check(self.cur.execute, "select 4") def test_closed_create_function(self): - con = sqlite.connect(":memory:") - con.close() - def f(x): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_function("foo", 1, f) + def f(x): + return 17 + self.check(self.con.create_function, "foo", 1, f) def test_closed_create_aggregate(self): - con = sqlite.connect(":memory:") - con.close() class Agg: def __init__(self): pass @@ -1727,29 +1718,21 @@ def step(self, x): pass def finalize(self): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_aggregate("foo", 1, Agg) + self.check(self.con.create_aggregate, "foo", 1, Agg) def test_closed_set_authorizer(self): - con = sqlite.connect(":memory:") - con.close() def authorizer(*args): return sqlite.DENY - with self.assertRaises(sqlite.ProgrammingError): - con.set_authorizer(authorizer) + self.check(self.con.set_authorizer, authorizer) def test_closed_set_progress_callback(self): - con = sqlite.connect(":memory:") - con.close() - def progress(): pass - with self.assertRaises(sqlite.ProgrammingError): - con.set_progress_handler(progress, 100) + def progress(): + pass + self.check(self.con.set_progress_handler, progress, 100) def test_closed_call(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con() + self.check(self.con) + class ClosedCurTests(unittest.TestCase): def test_closed(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index d0c24b9c60e613..5f6811fb5cc0a5 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -2,16 +2,12 @@ import unittest import sqlite3 as sqlite -from .test_dbapi import memory_database +from .util import memory_database +from .util import MemoryDatabaseMixin -class DumpTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cu = self.cx.cursor() - def tearDown(self): - self.cx.close() +class DumpTests(MemoryDatabaseMixin, unittest.TestCase): def test_table_dump(self): expected_sqls = [ diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index d63589483e1042..a7c4417862aff7 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -24,6 +24,9 @@ import sqlite3 as sqlite from collections.abc import Sequence +from .util import memory_database +from .util import MemoryDatabaseMixin + def dict_factory(cursor, row): d = {} @@ -45,10 +48,12 @@ class OkFactory(sqlite.Connection): def __init__(self, *args, **kwargs): sqlite.Connection.__init__(self, *args, **kwargs) - for factory in DefectFactory, OkFactory: - with self.subTest(factory=factory): - con = sqlite.connect(":memory:", factory=factory) - self.assertIsInstance(con, factory) + with memory_database(factory=OkFactory) as con: + self.assertIsInstance(con, OkFactory) + regex = "Base Connection.__init__ not called." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + with memory_database(factory=DefectFactory) as con: + self.assertIsInstance(con, DefectFactory) def test_connection_factory_relayed_call(self): # gh-95132: keyword args must not be passed as positional args @@ -57,9 +62,9 @@ def __init__(self, *args, **kwargs): kwargs["isolation_level"] = None super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", factory=Factory) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) + with memory_database(factory=Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) def test_connection_factory_as_positional_arg(self): class Factory(sqlite.Connection): @@ -74,18 +79,13 @@ def __init__(self, *args, **kwargs): r"parameters in Python 3.15." ) with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) + with memory_database(5.0, 0, None, True, Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) self.assertEqual(cm.filename, __file__) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) -class CursorFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - - def tearDown(self): - self.con.close() +class CursorFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_is_instance(self): cur = self.con.cursor() @@ -103,9 +103,8 @@ def test_invalid_factory(self): # invalid callable returning non-cursor self.assertRaises(TypeError, self.con.cursor, lambda con: None) -class RowFactoryTestsBackwardsCompat(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") + +class RowFactoryTestsBackwardsCompat(MemoryDatabaseMixin, unittest.TestCase): def test_is_produced_by_factory(self): cur = self.con.cursor(factory=MyCursor) @@ -114,12 +113,8 @@ def test_is_produced_by_factory(self): self.assertIsInstance(row, dict) cur.close() - def tearDown(self): - self.con.close() -class RowFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class RowFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_custom_factory(self): self.con.row_factory = lambda cur, row: list(row) @@ -265,12 +260,8 @@ class FakeCursor(str): self.assertRaises(TypeError, self.con.cursor, FakeCursor) self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ()) - def tearDown(self): - self.con.close() -class TextFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TextFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_unicode(self): austria = "Österreich" @@ -291,15 +282,17 @@ def test_custom(self): self.assertEqual(type(row[0]), str, "type of row[0] must be unicode") self.assertTrue(row[0].endswith("reich"), "column must contain original data") - def tearDown(self): - self.con.close() class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase): + def setUp(self): self.con = sqlite.connect(":memory:") self.con.execute("create table test (value text)") self.con.execute("insert into test (value) values (?)", ("a\x00b",)) + def tearDown(self): + self.con.close() + def test_string(self): # text_factory defaults to str row = self.con.execute("select value from test").fetchone() @@ -325,9 +318,6 @@ def test_custom(self): self.assertIs(type(row[0]), bytes) self.assertEqual(row[0], b"a\x00b") - def tearDown(self): - self.con.close() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 89230c08cc9143..33f0af99532a10 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -26,34 +26,31 @@ from test.support.os_helper import TESTFN, unlink -from test.test_sqlite3.test_dbapi import memory_database, cx_limit -from test.test_sqlite3.test_userfunctions import with_tracebacks +from .util import memory_database, cx_limit, with_tracebacks +from .util import MemoryDatabaseMixin -class CollationTests(unittest.TestCase): +class CollationTests(MemoryDatabaseMixin, unittest.TestCase): + def test_create_collation_not_string(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError): - con.create_collation(None, lambda x, y: (x > y) - (x < y)) + self.con.create_collation(None, lambda x, y: (x > y) - (x < y)) def test_create_collation_not_callable(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError) as cm: - con.create_collation("X", 42) + self.con.create_collation("X", 42) self.assertEqual(str(cm.exception), 'parameter must be callable') def test_create_collation_not_ascii(self): - con = sqlite.connect(":memory:") - con.create_collation("collä", lambda x, y: (x > y) - (x < y)) + self.con.create_collation("collä", lambda x, y: (x > y) - (x < y)) def test_create_collation_bad_upper(self): class BadUpperStr(str): def upper(self): return None - con = sqlite.connect(":memory:") mycoll = lambda x, y: -((x > y) - (x < y)) - con.create_collation(BadUpperStr("mycoll"), mycoll) - result = con.execute(""" + self.con.create_collation(BadUpperStr("mycoll"), mycoll) + result = self.con.execute(""" select x from ( select 'a' as x union @@ -68,8 +65,7 @@ def mycoll(x, y): # reverse order return -((x > y) - (x < y)) - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -79,21 +75,20 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg='the expected order was not returned') - con.create_collation("mycoll", None) + self.con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') def test_collation_returns_large_integer(self): def mycoll(x, y): # reverse order return -((x > y) - (x < y)) * 2**32 - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -103,7 +98,7 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg="the expected order was not returned") @@ -112,7 +107,7 @@ def test_collation_register_twice(self): Register two different collation functions under the same name. Verify that the last one is actually used. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", lambda x, y: -((x > y) - (x < y))) result = con.execute(""" @@ -126,25 +121,26 @@ def test_deregister_collation(self): Register a collation, then deregister it. Make sure an error is raised if we try to use it. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: con.execute("select 'a' as x union select 'b' as x order by x collate mycoll") self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') -class ProgressTests(unittest.TestCase): + +class ProgressTests(MemoryDatabaseMixin, unittest.TestCase): + def test_progress_handler_used(self): """ Test that the progress handler is invoked once it is set. """ - con = sqlite.connect(":memory:") progress_calls = [] def progress(): progress_calls.append(None) return 0 - con.set_progress_handler(progress, 1) - con.execute(""" + self.con.set_progress_handler(progress, 1) + self.con.execute(""" create table foo(a, b) """) self.assertTrue(progress_calls) @@ -153,7 +149,7 @@ def test_opcode_count(self): """ Test that the opcode argument is respected. """ - con = sqlite.connect(":memory:") + con = self.con progress_calls = [] def progress(): progress_calls.append(None) @@ -176,11 +172,10 @@ def test_cancel_operation(self): """ Test that returning a non-zero value stops the operation in progress. """ - con = sqlite.connect(":memory:") def progress(): return 1 - con.set_progress_handler(progress, 1) - curs = con.cursor() + self.con.set_progress_handler(progress, 1) + curs = self.con.cursor() self.assertRaises( sqlite.OperationalError, curs.execute, @@ -190,7 +185,7 @@ def test_clear_handler(self): """ Test that setting the progress handler to None clears the previously set handler. """ - con = sqlite.connect(":memory:") + con = self.con action = 0 def progress(): nonlocal action @@ -203,31 +198,30 @@ def progress(): @with_tracebacks(ZeroDivisionError, name="bad_progress") def test_error_in_progress_handler(self): - con = sqlite.connect(":memory:") def bad_progress(): 1 / 0 - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) @with_tracebacks(ZeroDivisionError, name="bad_progress") def test_error_in_progress_handler_result(self): - con = sqlite.connect(":memory:") class BadBool: def __bool__(self): 1 / 0 def bad_progress(): return BadBool() - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) -class TraceCallbackTests(unittest.TestCase): +class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): + @contextlib.contextmanager def check_stmt_trace(self, cx, expected): try: @@ -242,12 +236,11 @@ def test_trace_callback_used(self): """ Test that the trace callback is invoked once it is set. """ - con = sqlite.connect(":memory:") traced_statements = [] def trace(statement): traced_statements.append(statement) - con.set_trace_callback(trace) - con.execute("create table foo(a, b)") + self.con.set_trace_callback(trace) + self.con.execute("create table foo(a, b)") self.assertTrue(traced_statements) self.assertTrue(any("create table foo" in stmt for stmt in traced_statements)) @@ -255,7 +248,7 @@ def test_clear_trace_callback(self): """ Test that setting the trace callback to None clears the previously set callback. """ - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) @@ -269,7 +262,7 @@ def test_unicode_content(self): Test that the statement can contain unicode literals. """ unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac' - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 7e8221e7227e6e..db4e13222da9da 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -28,15 +28,12 @@ from test import support from unittest.mock import patch -from test.test_sqlite3.test_dbapi import memory_database, cx_limit +from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin -class RegressionTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - def tearDown(self): - self.con.close() +class RegressionTests(MemoryDatabaseMixin, unittest.TestCase): def test_pragma_user_version(self): # This used to crash pysqlite because this pragma command returns NULL for the column name @@ -45,28 +42,24 @@ def test_pragma_user_version(self): def test_pragma_schema_version(self): # This still crashed pysqlite <= 2.2.1 - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES) - try: + with memory_database(detect_types=sqlite.PARSE_COLNAMES) as con: cur = self.con.cursor() cur.execute("pragma schema_version") - finally: - cur.close() - con.close() def test_statement_reset(self): # pysqlite 2.1.0 to 2.2.0 have the problem that not all statements are # reset before a rollback, but only those that are still in the # statement cache. The others are not accessible from the connection object. - con = sqlite.connect(":memory:", cached_statements=5) - cursors = [con.cursor() for x in range(5)] - cursors[0].execute("create table test(x)") - for i in range(10): - cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) + with memory_database(cached_statements=5) as con: + cursors = [con.cursor() for x in range(5)] + cursors[0].execute("create table test(x)") + for i in range(10): + cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) - for i in range(5): - cursors[i].execute(" " * i + "select x from test") + for i in range(5): + cursors[i].execute(" " * i + "select x from test") - con.rollback() + con.rollback() def test_column_name_with_spaces(self): cur = self.con.cursor() @@ -81,17 +74,15 @@ def test_statement_finalization_on_close_db(self): # cache when closing the database. statements that were still # referenced in cursors weren't closed and could provoke " # "OperationalError: Unable to close due to unfinalised statements". - con = sqlite.connect(":memory:") cursors = [] # default statement cache size is 100 for i in range(105): - cur = con.cursor() + cur = self.con.cursor() cursors.append(cur) cur.execute("select 1 x union select " + str(i)) - con.close() def test_on_conflict_rollback(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table foo(x, unique(x) on conflict rollback)") con.execute("insert into foo(x) values (1)") try: @@ -126,16 +117,16 @@ def test_type_map_usage(self): a statement. This test exhibits the problem. """ SELECT = "select * from foo" - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("create table foo(bar timestamp)") - with self.assertWarnsRegex(DeprecationWarning, "adapter"): - cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) - cur.execute(SELECT) - cur.execute("drop table foo") - cur.execute("create table foo(bar integer)") - cur.execute("insert into foo(bar) values (5)") - cur.execute(SELECT) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("create table foo(bar timestamp)") + with self.assertWarnsRegex(DeprecationWarning, "adapter"): + cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) + cur.execute(SELECT) + cur.execute("drop table foo") + cur.execute("create table foo(bar integer)") + cur.execute("insert into foo(bar) values (5)") + cur.execute(SELECT) def test_bind_mutating_list(self): # Issue41662: Crash when mutate a list of parameters during iteration. @@ -144,11 +135,11 @@ def __conform__(self, protocol): parameters.clear() return "..." parameters = [X(), 0] - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - con.execute("create table foo(bar X, baz integer)") - # Should not crash - with self.assertRaises(IndexError): - con.execute("insert into foo(bar, baz) values (?, ?)", parameters) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + con.execute("create table foo(bar X, baz integer)") + # Should not crash + with self.assertRaises(IndexError): + con.execute("insert into foo(bar, baz) values (?, ?)", parameters) def test_error_msg_decode_error(self): # When porting the module to Python 3.0, the error message about @@ -173,7 +164,7 @@ def upper(self): def __del__(self): con.isolation_level = "" - con = sqlite.connect(":memory:") + con = self.con con.isolation_level = None for level in "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE": with self.subTest(level=level): @@ -204,8 +195,7 @@ class Cursor(sqlite.Cursor): def __init__(self, con): pass - con = sqlite.connect(":memory:") - cur = Cursor(con) + cur = Cursor(self.con) with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4+5").fetchall() with self.assertRaisesRegex(sqlite.ProgrammingError, @@ -238,7 +228,9 @@ def test_auto_commit(self): 2.5.3 introduced a regression so that these could no longer be created. """ - con = sqlite.connect(":memory:", isolation_level=None) + with memory_database(isolation_level=None) as con: + self.assertIsNone(con.isolation_level) + self.assertFalse(con.in_transaction) def test_pragma_autocommit(self): """ @@ -273,9 +265,7 @@ def test_recursive_cursor_use(self): Recursively using a cursor, such as when reusing it from a generator led to segfaults. Now we catch recursive cursor usage and raise a ProgrammingError. """ - con = sqlite.connect(":memory:") - - cur = con.cursor() + cur = self.con.cursor() cur.execute("create table a (bar)") cur.execute("create table b (baz)") @@ -295,29 +285,30 @@ def test_convert_timestamp_microsecond_padding(self): since the microsecond string "456" actually represents "456000". """ - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("CREATE TABLE t (x TIMESTAMP)") + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("CREATE TABLE t (x TIMESTAMP)") - # Microseconds should be 456000 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") + # Microseconds should be 456000 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") - # Microseconds should be truncated to 123456 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") + # Microseconds should be truncated to 123456 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") - cur.execute("SELECT * FROM t") - with self.assertWarnsRegex(DeprecationWarning, "converter"): - values = [x[0] for x in cur.fetchall()] + cur.execute("SELECT * FROM t") + with self.assertWarnsRegex(DeprecationWarning, "converter"): + values = [x[0] for x in cur.fetchall()] - self.assertEqual(values, [ - datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), - datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), - ]) + self.assertEqual(values, [ + datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), + datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), + ]) def test_invalid_isolation_level_type(self): # isolation level is a string, not an integer - self.assertRaises(TypeError, - sqlite.connect, ":memory:", isolation_level=123) + regex = "isolation_level must be str or None" + with self.assertRaisesRegex(TypeError, regex): + memory_database(isolation_level=123).__enter__() def test_null_character(self): @@ -333,7 +324,7 @@ def test_null_character(self): cur.execute, query) def test_surrogates(self): - con = sqlite.connect(":memory:") + con = self.con self.assertRaises(UnicodeEncodeError, con, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, con, "select '\udcff'") cur = con.cursor() @@ -359,7 +350,7 @@ def test_commit_cursor_reset(self): to return rows multiple times when fetched from cursors after commit. See issues 10513 and 23129 for details. """ - con = sqlite.connect(":memory:") + con = self.con con.executescript(""" create table t(c); create table t2(c); @@ -391,10 +382,9 @@ def test_bpo31770(self): """ def callback(*args): pass - con = sqlite.connect(":memory:") - cur = sqlite.Cursor(con) + cur = sqlite.Cursor(self.con) ref = weakref.ref(cur, callback) - cur.__init__(con) + cur.__init__(self.con) del cur # The interpreter shouldn't crash when ref is collected. del ref @@ -425,6 +415,7 @@ def test_return_empty_bytestring(self): def test_table_lock_cursor_replace_stmt(self): with memory_database() as con: + con = self.con cur = con.cursor() cur.execute("create table t(t)") cur.executemany("insert into t values(?)", diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py index 5d211dd47b0b6b..b7b231d2225852 100644 --- a/Lib/test/test_sqlite3/test_transactions.py +++ b/Lib/test/test_sqlite3/test_transactions.py @@ -28,7 +28,8 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok -from test.test_sqlite3.test_dbapi import memory_database +from .util import memory_database +from .util import MemoryDatabaseMixin TIMEOUT = LOOPBACK_TIMEOUT / 10 @@ -132,14 +133,14 @@ def test_locking(self): def test_rollback_cursor_consistency(self): """Check that cursors behave correctly after rollback.""" - con = sqlite.connect(":memory:") - cur = con.cursor() - cur.execute("create table test(x)") - cur.execute("insert into test(x) values (5)") - cur.execute("select 1 union select 2 union select 3") + with memory_database() as con: + cur = con.cursor() + cur.execute("create table test(x)") + cur.execute("insert into test(x) values (5)") + cur.execute("select 1 union select 2 union select 3") - con.rollback() - self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) + con.rollback() + self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) def test_multiple_cursors_and_iternext(self): # gh-94028: statements are cleared and reset in cursor iternext. @@ -218,10 +219,7 @@ def test_no_duplicate_rows_after_rollback_new_query(self): -class SpecialCommandTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - self.cur = self.con.cursor() +class SpecialCommandTests(MemoryDatabaseMixin, unittest.TestCase): def test_drop_table(self): self.cur.execute("create table test(i)") @@ -233,14 +231,8 @@ def test_pragma(self): self.cur.execute("insert into test(i) values (5)") self.cur.execute("pragma count_changes=1") - def tearDown(self): - self.cur.close() - self.con.close() - -class TransactionalDDL(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TransactionalDDL(MemoryDatabaseMixin, unittest.TestCase): def test_ddl_does_not_autostart_transaction(self): # For backwards compatibility reasons, DDL statements should not @@ -268,9 +260,6 @@ def test_transactional_ddl(self): with self.assertRaises(sqlite.OperationalError): self.con.execute("select * from test") - def tearDown(self): - self.con.close() - class IsolationLevelFromInit(unittest.TestCase): CREATE = "create table t(t)" diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 05c2fb3aa6f8f2..5d12636dcd2b63 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -21,54 +21,15 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. -import contextlib -import functools -import io -import re import sys import unittest import sqlite3 as sqlite from unittest.mock import Mock, patch -from test.support import bigmemtest, catch_unraisable_exception, gc_collect - -from test.test_sqlite3.test_dbapi import cx_limit - - -def with_tracebacks(exc, regex="", name=""): - """Convenience decorator for testing callback tracebacks.""" - def decorator(func): - _regex = re.compile(regex) if regex else None - @functools.wraps(func) - def wrapper(self, *args, **kwargs): - with catch_unraisable_exception() as cm: - # First, run the test with traceback enabled. - with check_tracebacks(self, cm, exc, _regex, name): - func(self, *args, **kwargs) - - # Then run the test with traceback disabled. - func(self, *args, **kwargs) - return wrapper - return decorator - - -@contextlib.contextmanager -def check_tracebacks(self, cm, exc, regex, obj_name): - """Convenience context manager for testing callback tracebacks.""" - sqlite.enable_callback_tracebacks(True) - try: - buf = io.StringIO() - with contextlib.redirect_stderr(buf): - yield - - self.assertEqual(cm.unraisable.exc_type, exc) - if regex: - msg = str(cm.unraisable.exc_value) - self.assertIsNotNone(regex.search(msg)) - if obj_name: - self.assertEqual(cm.unraisable.object.__name__, obj_name) - finally: - sqlite.enable_callback_tracebacks(False) +from test.support import bigmemtest, gc_collect + +from .util import cx_limit, memory_database +from .util import with_tracebacks, check_tracebacks def func_returntext(): @@ -405,19 +366,19 @@ def test_func_deterministic_keyword_only(self): def test_function_destructor_via_gc(self): # See bpo-44304: The destructor of the user function can # crash if is called without the GIL from the gc functions - dest = sqlite.connect(':memory:') def md5sum(t): return - dest.create_function("md5", 1, md5sum) - x = dest("create table lang (name, first_appeared)") - del md5sum, dest + with memory_database() as dest: + dest.create_function("md5", 1, md5sum) + x = dest("create table lang (name, first_appeared)") + del md5sum, dest - y = [x] - y.append(y) + y = [x] + y.append(y) - del x,y - gc_collect() + del x,y + gc_collect() @with_tracebacks(OverflowError) def test_func_return_too_large_int(self): @@ -514,6 +475,10 @@ def setUp(self): """ self.con.create_window_function("sumint", 1, WindowSumInt) + def tearDown(self): + self.cur.close() + self.con.close() + def test_win_sum_int(self): self.cur.execute(self.query % "sumint") self.assertEqual(self.cur.fetchall(), self.expected) @@ -634,6 +599,7 @@ def setUp(self): """) cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)", ("foo", 5, 3.14, None, memoryview(b"blob"),)) + cur.close() self.con.create_aggregate("nostep", 1, AggrNoStep) self.con.create_aggregate("nofinalize", 1, AggrNoFinalize) @@ -646,9 +612,7 @@ def setUp(self): self.con.create_aggregate("aggtxt", 1, AggrText) def tearDown(self): - #self.cur.close() - #self.con.close() - pass + self.con.close() def test_aggr_error_on_create(self): with self.assertRaises(sqlite.OperationalError): @@ -775,7 +739,7 @@ def setUp(self): self.con.set_authorizer(self.authorizer_cb) def tearDown(self): - pass + self.con.close() def test_table_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: diff --git a/Lib/test/test_sqlite3/util.py b/Lib/test/test_sqlite3/util.py new file mode 100644 index 00000000000000..505406c437b632 --- /dev/null +++ b/Lib/test/test_sqlite3/util.py @@ -0,0 +1,78 @@ +import contextlib +import functools +import io +import re +import sqlite3 +import test.support +import unittest + + +# Helper for temporary memory databases +def memory_database(*args, **kwargs): + cx = sqlite3.connect(":memory:", *args, **kwargs) + return contextlib.closing(cx) + + +# Temporarily limit a database connection parameter +@contextlib.contextmanager +def cx_limit(cx, category=sqlite3.SQLITE_LIMIT_SQL_LENGTH, limit=128): + try: + _prev = cx.setlimit(category, limit) + yield limit + finally: + cx.setlimit(category, _prev) + + +def with_tracebacks(exc, regex="", name=""): + """Convenience decorator for testing callback tracebacks.""" + def decorator(func): + _regex = re.compile(regex) if regex else None + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with test.support.catch_unraisable_exception() as cm: + # First, run the test with traceback enabled. + with check_tracebacks(self, cm, exc, _regex, name): + func(self, *args, **kwargs) + + # Then run the test with traceback disabled. + func(self, *args, **kwargs) + return wrapper + return decorator + + +@contextlib.contextmanager +def check_tracebacks(self, cm, exc, regex, obj_name): + """Convenience context manager for testing callback tracebacks.""" + sqlite3.enable_callback_tracebacks(True) + try: + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + yield + + self.assertEqual(cm.unraisable.exc_type, exc) + if regex: + msg = str(cm.unraisable.exc_value) + self.assertIsNotNone(regex.search(msg)) + if obj_name: + self.assertEqual(cm.unraisable.object.__name__, obj_name) + finally: + sqlite3.enable_callback_tracebacks(False) + + +class MemoryDatabaseMixin: + + def setUp(self): + self.con = sqlite3.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + + @property + def cx(self): + return self.con + + @property + def cu(self): + return self.cur From f51f0466c07eabc6177c2f64f70c952dada050e8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 17 Aug 2023 11:16:00 +0300 Subject: [PATCH 172/598] gh-107298: Fix some references in the C API documentation (GH-108072) --- Doc/c-api/typeobj.rst | 2 +- Doc/extending/extending.rst | 4 ++-- Doc/using/configure.rst | 4 ++-- Doc/whatsnew/2.2.rst | 2 +- Doc/whatsnew/2.3.rst | 2 +- Doc/whatsnew/2.4.rst | 2 +- Doc/whatsnew/2.6.rst | 6 +++--- Doc/whatsnew/2.7.rst | 12 ++++++------ Doc/whatsnew/3.0.rst | 2 +- Doc/whatsnew/3.1.rst | 2 +- Doc/whatsnew/3.2.rst | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index d394ce10504b0e..cd037b4de882e1 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1697,7 +1697,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) to a pointer, are valid C99 address constants. However, the unary '&' operator applied to a non-static variable - like :c:func:`PyBaseObject_Type` is not required to produce an address + like :c:data:`PyBaseObject_Type` is not required to produce an address constant. Compilers may support this (gcc does), MSVC does not. Both compilers are strictly standard conforming in this particular behavior. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index f58b4f28113e8c..1ee7f28b2ba220 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -242,7 +242,7 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this +We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. The :exc:`!spam.error` exception can be raised in your extension module using a @@ -363,7 +363,7 @@ only non-\ ``static`` item defined in the module file:: return PyModule_Create(&spammodule); } -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, +Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 441d346a1a38a0..f4adea82a87c38 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -746,8 +746,8 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and -:c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined +The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_DATA()` and +:c:macro:`PyMODINIT_FUNC` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: * Use ``Py_EXPORTED_SYMBOL`` if the ``Py_BUILD_CORE_MODULE`` is defined diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 44e9bd8d492bfc..7de48a40263034 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1109,7 +1109,7 @@ code, none of the changes described here will affect you very much. definition tables to simplify implementation of methods with no arguments or a single untyped argument. Calling such methods is more efficient than calling a corresponding method that uses :c:macro:`METH_VARARGS`. Also, the old - :c:macro:`METH_OLDARGS` style of writing C methods is now officially deprecated. + :c:macro:`!METH_OLDARGS` style of writing C methods is now officially deprecated. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index a96c1061455e00..2120ee49f7d6bd 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1886,7 +1886,7 @@ Changes to Python's build process and to the C API include: (:file:`libpython2.3.so`) by supplying :option:`!--enable-shared` when running Python's :program:`configure` script. (Contributed by Ondrej Palkovsky.) -* The :c:macro:`DL_EXPORT` and :c:macro:`DL_IMPORT` macros are now deprecated. +* The :c:macro:`!DL_EXPORT` and :c:macro:`!DL_IMPORT` macros are now deprecated. Initialization functions for Python extension modules should now be declared using the new macro :c:macro:`PyMODINIT_FUNC`, while the Python core will generally use the :c:macro:`PyAPI_FUNC` and :c:macro:`PyAPI_DATA` macros. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 9e8a9e6a622d00..40b404f87e5152 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1476,7 +1476,7 @@ Some of the changes to Python's build process and to the C API are: :c:func:`PyArg_ParseTupleAndKeywords` but takes a :c:type:`va_list` instead of a number of arguments. (Contributed by Greg Chapman.) -* A new method flag, :c:macro:`METH_COEXISTS`, allows a function defined in slots +* A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve the access time for a method such as :meth:`set.__contains__`. (Contributed by Raymond Hettinger.) diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index ad899d53886c59..5c46dc8952154b 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -3060,9 +3060,9 @@ Changes to Python's build process and to the C API include: * Some macros were renamed in both 3.0 and 2.6 to make it clearer that they are macros, - not functions. :c:macro:`Py_Size()` became :c:macro:`Py_SIZE()`, - :c:macro:`Py_Type()` became :c:macro:`Py_TYPE()`, and - :c:macro:`Py_Refcnt()` became :c:macro:`Py_REFCNT()`. + not functions. :c:macro:`!Py_Size()` became :c:macro:`Py_SIZE()`, + :c:macro:`!Py_Type()` became :c:macro:`Py_TYPE()`, and + :c:macro:`!Py_Refcnt()` became :c:macro:`Py_REFCNT()`. The mixed-case macros are still available in Python 2.6 for backward compatibility. (:issue:`1629`) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 4b5a31f8a84810..00001f92b51c82 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2287,10 +2287,10 @@ object, and then get the ``void *`` pointer, which will usually point to an array of pointers to the module's various API functions. There is an existing data type already used for this, -:c:type:`PyCObject`, but it doesn't provide type safety. Evil code +:c:type:`!PyCObject`, but it doesn't provide type safety. Evil code written in pure Python could cause a segmentation fault by taking a -:c:type:`PyCObject` from module A and somehow substituting it for the -:c:type:`PyCObject` in module B. Capsules know their own name, +:c:type:`!PyCObject` from module A and somehow substituting it for the +:c:type:`!PyCObject` in module B. Capsules know their own name, and getting the pointer requires providing the name: .. code-block:: c @@ -2310,10 +2310,10 @@ detect the mismatched name and return false. Refer to :ref:`using-capsules` for more information on using these objects. Python 2.7 now uses capsules internally to provide various -extension-module APIs, but the :c:func:`PyCObject_AsVoidPtr` was +extension-module APIs, but the :c:func:`!PyCObject_AsVoidPtr` was modified to handle capsules, preserving compile-time compatibility -with the :c:type:`CObject` interface. Use of -:c:func:`PyCObject_AsVoidPtr` will signal a +with the :c:type:`!PyCObject` interface. Use of +:c:func:`!PyCObject_AsVoidPtr` will signal a :exc:`PendingDeprecationWarning`, which is silent by default. Implemented in Python 3.1 and backported to 2.7 by Larry Hastings; diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index b767bbe177abeb..58d42bd94cb61e 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -875,7 +875,7 @@ to the C API. * Renamed the boolean conversion C-level slot and method: ``nb_nonzero`` is now ``nb_bool``. -* Removed :c:macro:`METH_OLDARGS` and :c:macro:`WITH_CYCLE_GC` from the C API. +* Removed :c:macro:`!METH_OLDARGS` and :c:macro:`!WITH_CYCLE_GC` from the C API. .. ====================================================================== diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index c399f007fd63fb..3c1c9c3c4bc601 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -510,7 +510,7 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson; :issue:`5914`.) -* Added :c:type:`PyCapsule` as a replacement for the :c:type:`PyCObject` API. +* Added :c:type:`PyCapsule` as a replacement for the :c:type:`!PyCObject` API. The principal difference is that the new type has a well defined interface for passing typing safety information and a less complicated signature for calling a destructor. The old type had a problematic API and is now diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index ed1c1770fb0f51..56ac5c4c0a1c1b 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2658,7 +2658,7 @@ require changes to your code: * "t#" format has been removed: use "s#" or "s*" instead * "w" and "w#" formats has been removed: use "w*" instead -* The :c:type:`PyCObject` type, deprecated in 3.1, has been removed. To wrap +* The :c:type:`!PyCObject` type, deprecated in 3.1, has been removed. To wrap opaque C pointers in Python objects, the :c:type:`PyCapsule` API should be used instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. From 33e6e3fec02ff3035dec52692542d3dd10124bef Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:01:14 +0100 Subject: [PATCH 173/598] GH-107987: Remove the Distributing Python Modules guide (#108016) --- Doc/conf.py | 2 - Doc/contents.rst | 1 - Doc/distributing/index.rst | 175 +++---------------------------------- Doc/installing/index.rst | 4 +- Doc/tutorial/venv.rst | 4 +- 5 files changed, 16 insertions(+), 170 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 19eab1a9340e47..16fd8bf257179e 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -354,8 +354,6 @@ latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), ('installing/index', 'installing.tex', diff --git a/Doc/contents.rst b/Doc/contents.rst index 649a1344a0b26a..24ceacb0076b5e 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,7 +11,6 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst installing/index.rst howto/index.rst faq/index.rst diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index d237f8f082d87b..2430564b45d6d2 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -1,174 +1,19 @@ +:orphan: + +.. This page is retained solely for existing links to /distributing/index.html. + Direct readers to the PPUG instead. + .. _distributing-index: ############################### Distributing Python Modules ############################### -:Email: distutils-sig@python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide `. - .. note:: - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Package Index `__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of ``distutils`` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for ``distutils`` first - published in 2004. Its most notable addition over the unmodified - ``distutils`` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to ``distutils`` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to ``distutils``/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.io/ - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including macOS and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations - -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _publishing-python-packages: - -Reading the Python Packaging User Guide -======================================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating and publishing a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Package Index`_ -* `The .pypirc file`_ - -.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Package Index to see if the name is already in use -* check popular hosting sites like GitHub, Bitbucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ - -.. other topics: + Information and guidance on distributing Python modules and packages + has been moved to the `Python Packaging User Guide`_, + and the tutorial on `packaging Python projects`_. - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + .. _Python Packaging User Guide: https://packaging.python.org/ + .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 5aec5178d48f3d..a46c1caefe4d8a 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -19,7 +19,9 @@ solutions to the common pool. This guide covers the installation part of the process. For a guide to creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. +`Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ .. note:: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index d1bba098d7d23b..a6dead2eac11f6 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -207,4 +207,6 @@ necessary packages with ``install -r``: ``pip`` has many more options. Consult the :ref:`installing-index` guide for complete documentation for ``pip``. When you've written a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. +consult the `Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ From 006e44f9502308ec3d14424ad8bd774046f2be8e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 17 Aug 2023 11:16:03 +0100 Subject: [PATCH 174/598] GH-108035: Remove the `_PyCFrame` struct as it is no longer needed for performance. (GH-108036) --- Include/cpython/pystate.h | 25 ++------------ Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_runtime.h | 2 +- Include/internal/pycore_runtime_init.h | 6 +--- ...-08-11-16-18-19.gh-issue-108035.e2msOD.rst | 4 +++ Modules/_xxsubinterpretersmodule.c | 2 +- Objects/genobject.c | 8 ++--- Objects/typevarobject.c | 2 +- Python/bytecodes.c | 34 ++++++++----------- Python/ceval.c | 22 ++++-------- Python/ceval_macros.h | 2 +- Python/executor.c | 2 +- Python/executor_cases.c.h | 3 +- Python/frame.c | 2 +- Python/generated_cases.c.h | 33 ++++++++---------- Python/instrumentation.c | 2 +- Python/intrinsics.c | 4 +-- Python/pylifecycle.c | 2 +- Python/pystate.c | 6 ++-- Python/sysmodule.c | 4 +-- Python/traceback.c | 2 +- 21 files changed, 66 insertions(+), 103 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 56e473cc5e42d5..2a53838314c9d7 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -29,24 +29,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 -// Internal structure: you should not use it directly, but use public functions -// like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). -typedef struct _PyCFrame { - /* This struct will be threaded through the C stack - * allowing fast access to per-thread state that needs - * to be accessed quickly by the interpreter, but can - * be modified outside of the interpreter. - * - * WARNING: This makes data on the C stack accessible from - * heap objects. Care must be taken to maintain stack - * discipline and make sure that instances of this struct cannot - * accessed outside of their lifetime. - */ - /* Pointer to the currently executing frame (it can be NULL) */ - struct _PyInterpreterFrame *current_frame; - struct _PyCFrame *previous; -} _PyCFrame; - typedef struct _err_stackitem { /* This struct represents a single execution context where we might * be currently handling an exception. It is a per-coroutine state @@ -123,9 +105,8 @@ struct _ts { int tracing; int what_event; /* The event currently being monitored, if any. */ - /* Pointer to current _PyCFrame in the C stack frame of the currently, - * or most recently, executing _PyEval_EvalFrameDefault. */ - _PyCFrame *cframe; + /* Pointer to currently executing frame. */ + struct _PyInterpreterFrame *current_frame; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; @@ -211,8 +192,6 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; - /* The bottom-most frame on the stack. */ - _PyCFrame root_cframe; }; /* WASI has limited call stack. Python's recursion limit depends on code diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 5ff20ef845ab10..0dc2a1814cb1d5 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -196,7 +196,7 @@ _PyFrame_GetFirstComplete(_PyInterpreterFrame *frame) static inline _PyInterpreterFrame * _PyThreadState_GetFrame(PyThreadState *tstate) { - return _PyFrame_GetFirstComplete(tstate->cframe->current_frame); + return _PyFrame_GetFirstComplete(tstate->current_frame); } /* For use by _PyFrame_GetFrameObject diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0ec86ee6c50ca3..63b485027486bb 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -80,7 +80,7 @@ typedef struct _Py_DebugOffsets { off_t prev; off_t next; off_t interp; - off_t cframe; + off_t current_frame; off_t thread_id; off_t native_thread_id; } thread_state; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e89d368be07aa7..ea29c69f59acc7 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -45,7 +45,7 @@ extern PyTypeObject _PyExc_MemoryError; .prev = offsetof(PyThreadState, prev), \ .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ - .cframe = offsetof(PyThreadState, cframe), \ + .current_frame = offsetof(PyThreadState, current_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ }, \ @@ -56,10 +56,6 @@ extern PyTypeObject _PyExc_MemoryError; .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ }, \ - .cframe = { \ - .current_frame = offsetof(_PyCFrame, current_frame), \ - .previous = offsetof(_PyCFrame, previous), \ - }, \ .code_object = { \ .filename = offsetof(PyCodeObject, co_filename), \ .name = offsetof(PyCodeObject, co_name), \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst new file mode 100644 index 00000000000000..fc2369ddabb83c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst @@ -0,0 +1,4 @@ +Remove the ``_PyCFrame`` struct, moving the pointer to the current intepreter frame +back to the threadstate, as it was for 3.10 and earlier. The ``_PyCFrame`` +existed as a performance optimization for tracing. Since PEP 669 has been +implemented, this optimization no longer applies. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 31373f8fdf8c71..ea91e70cad991d 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -369,7 +369,7 @@ _is_running(PyInterpreterState *interp) } assert(!PyErr_Occurred()); - struct _PyInterpreterFrame *frame = tstate->cframe->current_frame; + struct _PyInterpreterFrame *frame = tstate->current_frame; if (frame == NULL) { return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 65782be182cd71..10c55efb1b1e9b 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -476,9 +476,9 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, will be reported correctly to the user. */ /* XXX We should probably be updating the current frame somewhere in ceval.c. */ - _PyInterpreterFrame *prev = tstate->cframe->current_frame; + _PyInterpreterFrame *prev = tstate->current_frame; frame->previous = prev; - tstate->cframe->current_frame = frame; + tstate->current_frame = frame; /* Close the generator that we are currently iterating with 'yield from' or awaiting on with 'await'. */ PyFrameState state = gen->gi_frame_state; @@ -486,7 +486,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); gen->gi_frame_state = state; - tstate->cframe->current_frame = prev; + tstate->current_frame = prev; frame->previous = NULL; } else { /* `yf` is an iterator or a coroutine-like object. */ @@ -938,7 +938,7 @@ _Py_MakeCoro(PyFunctionObject *func) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index e09e6a62553cff..66122e36283ad0 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -107,7 +107,7 @@ make_union(PyObject *self, PyObject *other) static PyObject * caller(void) { - _PyInterpreterFrame *f = _PyThreadState_GET()->cframe->current_frame; + _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; if (f == NULL) { Py_RETURN_NONE; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ae2923c65b3308..6f17472e04e5e3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -72,7 +72,6 @@ dummy_func( _PyInterpreterFrame *frame, unsigned char opcode, unsigned int oparg, - _PyCFrame cframe, _Py_CODEUNIT *next_instr, PyObject **stack_pointer, PyObject *kwnames, @@ -134,8 +133,7 @@ dummy_func( } inst(RESUME, (--)) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); @@ -752,9 +750,8 @@ dummy_func( inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; @@ -768,7 +765,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -787,7 +784,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -803,7 +800,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -823,7 +820,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1019,7 +1016,7 @@ dummy_func( gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -1038,7 +1035,7 @@ dummy_func( gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -2207,10 +2204,10 @@ dummy_func( OBJECT_STAT_INC(optimization_attempts); frame = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) -1); goto resume_frame; } @@ -2238,7 +2235,7 @@ dummy_func( Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } goto resume_frame; @@ -2993,12 +2990,11 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; #if TIER_ONE - frame = cframe.current_frame = new_frame; goto start_frame; #endif #if TIER_TWO - frame = tstate->cframe->current_frame = new_frame; ERROR_IF(_Py_EnterRecursivePy(tstate), exit_unwind); stack_pointer = _PyFrame_GetStackPointer(frame); ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; @@ -3135,7 +3131,7 @@ dummy_func( /* Link frames */ init_frame->previous = shim; shim->previous = frame; - frame = cframe.current_frame = init_frame; + frame = tstate->current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -3598,7 +3594,7 @@ dummy_func( assert(frame != &entry_frame); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; + frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; } diff --git a/Python/ceval.c b/Python/ceval.c index 26e741ed7c7547..1e2262c1f18c3e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -656,17 +656,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int lltrace = 0; #endif - _PyCFrame cframe; _PyInterpreterFrame entry_frame; PyObject *kwnames = NULL; // Borrowed reference. Reset by CALL instructions. - /* WARNING: Because the _PyCFrame lives on the C stack, - * but can be accessed from a heap allocated object (tstate) - * strict stack discipline must be maintained. - */ - _PyCFrame *prev_cframe = tstate->cframe; - cframe.previous = prev_cframe; - tstate->cframe = &cframe; + #ifdef Py_DEBUG /* Set these to invalid but identifiable values for debugging. */ @@ -682,9 +675,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; /* Push frame */ - entry_frame.previous = prev_cframe->current_frame; + entry_frame.previous = tstate->current_frame; frame->previous = &entry_frame; - cframe.current_frame = frame; + tstate->current_frame = frame; tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { @@ -924,13 +917,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame == &entry_frame) { - /* Restore previous cframe and exit */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } @@ -2297,7 +2289,7 @@ int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = tstate->cframe->current_frame; + _PyInterpreterFrame *current_frame = tstate->current_frame; int result = cf->cf_flags != 0; if (current_frame != NULL) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 5e2db1e0b394e6..08f19cd9a397f1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -109,7 +109,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); \ frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ - frame = cframe.current_frame = (NEW_FRAME); \ + frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ } while (0) diff --git a/Python/executor.c b/Python/executor.c index 5a571e6da4673f..88c039da8539e2 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -111,7 +111,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject STACK_SHRINK(1); error: // On ERROR_IF we return NULL as the frame. - // The caller recovers the frame from cframe.current_frame. + // The caller recovers the frame from tstate->current_frame. DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b3dd3133530562..9fbf026f164a60 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2212,12 +2212,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; #if TIER_ONE - frame = cframe.current_frame = new_frame; goto start_frame; #endif #if TIER_TWO - frame = tstate->cframe->current_frame = new_frame; if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; stack_pointer = _PyFrame_GetStackPointer(frame); ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; diff --git a/Python/frame.c b/Python/frame.c index 581e4f95710fe2..fbfa54398c72b6 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -124,7 +124,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); // GH-99729: Clearing this frame can expose the stack (via finalizers). It's // crucial that this frame has been unlinked, and is no longer visible: - assert(_PyThreadState_GET()->cframe->current_frame != frame); + assert(_PyThreadState_GET()->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 11d560a6e77adf..80af8a7bcd56df 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,8 +8,7 @@ } TARGET(RESUME) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); @@ -961,9 +960,8 @@ retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; @@ -980,7 +978,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1002,7 +1000,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1019,7 +1017,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1039,7 +1037,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1263,7 +1261,7 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -1284,7 +1282,7 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -2911,10 +2909,10 @@ OBJECT_STAT_INC(optimization_attempts); frame = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) -1); goto resume_frame; } @@ -2933,7 +2931,7 @@ Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } goto resume_frame; @@ -3830,12 +3828,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; #if TIER_ONE - frame = cframe.current_frame = new_frame; goto start_frame; #endif #if TIER_TWO - frame = tstate->cframe->current_frame = new_frame; if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; stack_pointer = _PyFrame_GetStackPointer(frame); ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; @@ -4014,7 +4011,7 @@ /* Link frames */ init_frame->previous = shim; shim->previous = frame; - frame = cframe.current_frame = init_frame; + frame = tstate->current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -4636,7 +4633,7 @@ assert(frame != &entry_frame); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; + frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 48befed4ea838c..a7a5b4a5dc5f6e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1677,7 +1677,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { PyThreadState* ts = PyInterpreterState_ThreadHead(interp); HEAD_UNLOCK(runtime); while (ts) { - _PyInterpreterFrame *frame = ts->cframe->current_frame; + _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner != FRAME_OWNED_BY_CSTACK) { if (_Py_Instrument(_PyFrame_GetCode(frame), interp)) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 61a8e75872d2e2..5267c10238e626 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -120,7 +120,7 @@ import_all_from(PyThreadState *tstate, PyObject *locals, PyObject *v) static PyObject * import_star(PyThreadState* tstate, PyObject *from) { - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; if (_PyFrame_FastToLocalsWithError(frame) < 0) { return NULL; } @@ -142,7 +142,7 @@ import_star(PyThreadState* tstate, PyObject *from) static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; assert(frame->owner == FRAME_OWNED_BY_GENERATOR); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0de3abf9407899..90633960cb00ee 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2147,7 +2147,7 @@ Py_EndInterpreter(PyThreadState *tstate) if (tstate != _PyThreadState_GET()) { Py_FatalError("thread is not current"); } - if (tstate->cframe->current_frame != NULL) { + if (tstate->current_frame != NULL) { Py_FatalError("thread still has a frame"); } interp->finalizing = 1; diff --git a/Python/pystate.c b/Python/pystate.c index 3a05cb0fa7988d..01651d79f9acc0 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1310,7 +1310,7 @@ init_threadstate(PyThreadState *tstate, // This is cleared when PyGILState_Ensure() creates the thread state. tstate->gilstate_counter = 1; - tstate->cframe = &tstate->root_cframe; + tstate->current_frame = NULL; tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; @@ -1452,7 +1452,7 @@ PyThreadState_Clear(PyThreadState *tstate) int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose && tstate->cframe->current_frame != NULL) { + if (verbose && tstate->current_frame != NULL) { /* bpo-20526: After the main thread calls _PyInterpreterState_SetFinalizing() in Py_FinalizeEx() (or in Py_EndInterpreter() for subinterpreters), @@ -1953,7 +1953,7 @@ _PyThread_CurrentFrames(void) for (i = runtime->interpreters.head; i != NULL; i = i->next) { PyThreadState *t; for (t = i->threads.head; t != NULL; t = t->next) { - _PyInterpreterFrame *frame = t->cframe->current_frame; + _PyInterpreterFrame *frame = t->current_frame; frame = _PyFrame_GetFirstComplete(frame); if (frame == NULL) { continue; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index be026d95ba7e77..f82901181f8866 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1954,7 +1954,7 @@ sys__getframe_impl(PyObject *module, int depth) /*[clinic end generated code: output=d438776c04d59804 input=c1be8a6464b11ee5]*/ { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; if (frame != NULL) { while (depth > 0) { @@ -2270,7 +2270,7 @@ sys__getframemodulename_impl(PyObject *module, int depth) if (PySys_Audit("sys._getframemodulename", "i", depth) < 0) { return NULL; } - _PyInterpreterFrame *f = _PyThreadState_GET()->cframe->current_frame; + _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; while (f && (_PyFrame_IsIncomplete(f) || depth-- > 0)) { f = f->previous; } diff --git a/Python/traceback.c b/Python/traceback.c index ca524b1b9af78b..bddb8763a2f9be 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1207,7 +1207,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) PUTS(fd, "Stack (most recent call first):\n"); } - frame = tstate->cframe->current_frame; + frame = tstate->current_frame; if (frame == NULL) { PUTS(fd, " \n"); return; From 2da67a5789a9247ae6611185bf786b697aa04fbe Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:25:28 +0100 Subject: [PATCH 175/598] gh-105481: fix out of date comment (#108079) --- Tools/build/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 8dbb7bfa69a926..996d5f65a923b2 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -21,7 +21,7 @@ verbose = False -# This must be kept in sync with opcode.py +# This must be kept in sync with Tools/cases_generator/generate_cases.py RESUME = 166 def isprintable(b: bytes) -> bool: From 8a19f225b948db1eebe1d9fc71a486258841f578 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 17 Aug 2023 16:14:01 +0200 Subject: [PATCH 176/598] gh-107801: Document SEEK_HOLE and SEEK_DATA (#107936) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Antoine Pitrou --- Doc/library/os.rst | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 9735baa5bc0f3a..94b76e4052b94b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1171,6 +1171,10 @@ as internal buffering of data. current position; :const:`SEEK_END` or ``2`` to set it relative to the end of the file. Return the new cursor position in bytes, starting from the beginning. + .. versionchanged:: 3.3 + + Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + .. data:: SEEK_SET SEEK_CUR @@ -1179,9 +1183,30 @@ as internal buffering of data. Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, respectively. + +.. data:: SEEK_HOLE + SEEK_DATA + + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on file objects, for seeking file data and holes on sparsely + allocated files. + + :data:`!SEEK_DATA` + Adjust the file offset to the next location containing data, + relative to the seek position. + + :data:`!SEEK_HOLE` + Adjust the file offset to the next location containing a hole, + relative to the seek position. + A hole is defined as a sequence of zeros. + + .. note:: + + These operations only make sense for filesystems that support them. + + .. availability:: Linux >= 3.1, macOS, Unix + .. versionadded:: 3.3 - Some operating systems could support additional values, like - :const:`os.SEEK_HOLE` or :const:`os.SEEK_DATA`. .. function:: open(path, flags, mode=0o777, *, dir_fd=None) From 4cb08188e8f0faaec303e268e12aa1d6f54017f7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:37:07 +0100 Subject: [PATCH 177/598] Add workflow for automatic issue headers (#108054) We don't get the "Bug report" and "Feature or enhancement" titles anymore, with the new issue forms. This brings them back! --- .github/workflows/add-issue-header.yml | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/add-issue-header.yml diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml new file mode 100644 index 00000000000000..1ef9178b95e5f6 --- /dev/null +++ b/.github/workflows/add-issue-header.yml @@ -0,0 +1,53 @@ +name: Add issue header +# Automatically edits an issue's descriptions with a header, +# one of: +# +# - Bug report +# - Crash report +# - Feature or enhancement + +on: + issues: + types: + # Only ever run once + - opened + + +jobs: + add-header: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/github-script@v6 + with: + # language=JavaScript + script: | + // https://devguide.python.org/triage/labels/#type-labels + const HEADERS = new Map([ + ['type-bug', 'Bug report'], + ['type-crash', 'Crash report'], + ['type-feature', 'Feature or enhancement'], + ]); + let issue_data = await github.rest.issues.get({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(issue => issue.data); + let header = ''; + for (const label_data of issue_data.labels) { + const label_name = (typeof label_data === 'string') ? label_data : label_data.name; + if (HEADERS.has(label_name)) { + header = HEADERS.get(label_name); + break; + } + } + if (header !== '') { + console.log(`Setting new header: ${header}`); + await github.rest.issues.update({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `# ${header}\n\n${issue_data.body.replaceAll('\r', '')}` + }); + } From 0b243c2f665e6784b74ac4d602d4ee429a2ad7f0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:07:58 +0100 Subject: [PATCH 178/598] gh-105481: opcode.h is no longer generated during the build (#108080) --- Include/opcode.h | 3 +- Lib/opcode.py | 31 +------------- Makefile.pre.in | 4 +- Modules/_opcode.c | 70 ++++++++++++++++++++++++++++++++ Modules/clinic/_opcode.c.h | 22 +++++++++- PCbuild/regen.targets | 4 +- Tools/build/generate_opcode_h.py | 37 ++--------------- 7 files changed, 99 insertions(+), 72 deletions(-) diff --git a/Include/opcode.h b/Include/opcode.h index e5c42d5a718286..2619b690019acc 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -1,5 +1,3 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - #ifndef Py_OPCODE_H #define Py_OPCODE_H #ifdef __cplusplus @@ -36,6 +34,7 @@ extern "C" { #define NB_INPLACE_TRUE_DIVIDE 24 #define NB_INPLACE_XOR 25 +#define NB_OPARG_LAST 25 #ifdef __cplusplus } diff --git a/Lib/opcode.py b/Lib/opcode.py index 6b9d9ce811a61c..f8487522bfdc69 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -44,39 +44,10 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() + _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] -_nb_ops = [ - ("NB_ADD", "+"), - ("NB_AND", "&"), - ("NB_FLOOR_DIVIDE", "//"), - ("NB_LSHIFT", "<<"), - ("NB_MATRIX_MULTIPLY", "@"), - ("NB_MULTIPLY", "*"), - ("NB_REMAINDER", "%"), - ("NB_OR", "|"), - ("NB_POWER", "**"), - ("NB_RSHIFT", ">>"), - ("NB_SUBTRACT", "-"), - ("NB_TRUE_DIVIDE", "/"), - ("NB_XOR", "^"), - ("NB_INPLACE_ADD", "+="), - ("NB_INPLACE_AND", "&="), - ("NB_INPLACE_FLOOR_DIVIDE", "//="), - ("NB_INPLACE_LSHIFT", "<<="), - ("NB_INPLACE_MATRIX_MULTIPLY", "@="), - ("NB_INPLACE_MULTIPLY", "*="), - ("NB_INPLACE_REMAINDER", "%="), - ("NB_INPLACE_OR", "|="), - ("NB_INPLACE_POWER", "**="), - ("NB_INPLACE_RSHIFT", ">>="), - ("NB_INPLACE_SUBTRACT", "-="), - ("NB_INPLACE_TRUE_DIVIDE", "/="), - ("NB_INPLACE_XOR", "^="), -] - - _cache_format = { "LOAD_GLOBAL": { "counter": 1, diff --git a/Makefile.pre.in b/Makefile.pre.in index bcec0782f6e95e..9be5c3b50eb9ee 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1427,13 +1427,11 @@ regen-ast: .PHONY: regen-opcode regen-opcode: - # Regenerate Include/opcode.h from Lib/opcode.py + # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py # using Tools/build/generate_opcode_h.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ - $(srcdir)/Include/opcode.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new .PHONY: regen-token diff --git a/Modules/_opcode.c b/Modules/_opcode.c index ad0fa736f82767..4f85e7ee26de2a 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -223,6 +223,75 @@ _opcode_get_specialization_stats_impl(PyObject *module) /*[clinic input] +_opcode.get_nb_ops + +Return array of symbols of binary ops. + +Indexed by the BINARY_OP oparg value. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_nb_ops_impl(PyObject *module) +/*[clinic end generated code: output=d997d306cc15426f input=9462fc544c823176]*/ +{ + PyObject *list = PyList_New(NB_OPARG_LAST + 1); + if (list == NULL) { + return NULL; + } +#define ADD_NB_OP(NUM, STR) \ + do { \ + PyObject *pair = Py_BuildValue( \ + "NN", PyUnicode_FromString(#NUM), PyUnicode_FromString(STR)); \ + if (pair == NULL) { \ + Py_DECREF(list); \ + return NULL; \ + } \ + PyList_SET_ITEM(list, (NUM), pair); \ + } while(0); + + ADD_NB_OP(NB_ADD, "+"); + ADD_NB_OP(NB_AND, "&"); + ADD_NB_OP(NB_FLOOR_DIVIDE, "//"); + ADD_NB_OP(NB_LSHIFT, "<<"); + ADD_NB_OP(NB_MATRIX_MULTIPLY, "@"); + ADD_NB_OP(NB_MULTIPLY, "*"); + ADD_NB_OP(NB_REMAINDER, "%"); + ADD_NB_OP(NB_OR, "|"); + ADD_NB_OP(NB_POWER, "**"); + ADD_NB_OP(NB_RSHIFT, ">>"); + ADD_NB_OP(NB_SUBTRACT, "-"); + ADD_NB_OP(NB_TRUE_DIVIDE, "/"); + ADD_NB_OP(NB_XOR, "^"); + ADD_NB_OP(NB_INPLACE_ADD, "+="); + ADD_NB_OP(NB_INPLACE_AND, "&="); + ADD_NB_OP(NB_INPLACE_FLOOR_DIVIDE, "//="); + ADD_NB_OP(NB_INPLACE_LSHIFT, "<<="); + ADD_NB_OP(NB_INPLACE_MATRIX_MULTIPLY, "@="); + ADD_NB_OP(NB_INPLACE_MULTIPLY, "*="); + ADD_NB_OP(NB_INPLACE_REMAINDER, "%="); + ADD_NB_OP(NB_INPLACE_OR, "|="); + ADD_NB_OP(NB_INPLACE_POWER, "**="); + ADD_NB_OP(NB_INPLACE_RSHIFT, ">>="); + ADD_NB_OP(NB_INPLACE_SUBTRACT, "-="); + ADD_NB_OP(NB_INPLACE_TRUE_DIVIDE, "/="); + ADD_NB_OP(NB_INPLACE_XOR, "^="); + +#undef ADD_NB_OP + + for(int i = 0; i <= NB_OPARG_LAST; i++) { + if (PyList_GET_ITEM(list, i) == NULL) { + Py_DECREF(list); + PyErr_Format(PyExc_ValueError, + "Missing initialization for NB_OP %d", + i); + return NULL; + } + } + return list; +} + +/*[clinic input] + _opcode.get_intrinsic1_descs Return a list of names of the unary intrinsics. @@ -287,6 +356,7 @@ opcode_functions[] = { _OPCODE_HAS_LOCAL_METHODDEF _OPCODE_HAS_EXC_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF + _OPCODE_GET_NB_OPS_METHODDEF _OPCODE_GET_INTRINSIC1_DESCS_METHODDEF _OPCODE_GET_INTRINSIC2_DESCS_METHODDEF {NULL, NULL, 0, NULL} diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index e1fc5ba17f7078..52fbdcbd2a6528 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -613,6 +613,26 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) return _opcode_get_specialization_stats_impl(module); } +PyDoc_STRVAR(_opcode_get_nb_ops__doc__, +"get_nb_ops($module, /)\n" +"--\n" +"\n" +"Return array of symbols of binary ops.\n" +"\n" +"Indexed by the BINARY_OP oparg value."); + +#define _OPCODE_GET_NB_OPS_METHODDEF \ + {"get_nb_ops", (PyCFunction)_opcode_get_nb_ops, METH_NOARGS, _opcode_get_nb_ops__doc__}, + +static PyObject * +_opcode_get_nb_ops_impl(PyObject *module); + +static PyObject * +_opcode_get_nb_ops(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _opcode_get_nb_ops_impl(module); +} + PyDoc_STRVAR(_opcode_get_intrinsic1_descs__doc__, "get_intrinsic1_descs($module, /)\n" "--\n" @@ -648,4 +668,4 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=d85de5f2887b3661 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31a1a11c2f81dca4 input=a9049054013a1b77]*/ diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index ed02f9163549b9..2ff18a8966f557 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -14,7 +14,7 @@ -C <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h" /> + <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 344709a05184ce..643918c1bc2ce4 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -1,4 +1,4 @@ -# This script generates the opcode.h header file. +# This script generates the pycore_opcode.h header file. import sys import tokenize @@ -6,27 +6,6 @@ SCRIPT_NAME = "Tools/build/generate_opcode_h.py" PYTHON_OPCODE = "Lib/opcode.py" -opcode_h_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_OPCODE_H -#define Py_OPCODE_H -#ifdef __cplusplus -extern "C" {{ -#endif - -#include "opcode_ids.h" - -""".lstrip() - -opcode_h_footer = """ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_H */ -""" - internal_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} @@ -62,20 +41,10 @@ def get_python_module_dict(filename): return mod def main(opcode_py, - opcode_h='Include/opcode.h', internal_opcode_h='Include/internal/pycore_opcode.h'): opcode = get_python_module_dict(opcode_py) - with open(opcode_h, 'w') as fobj: - fobj.write(opcode_h_header) - - fobj.write("\n") - for i, (op, _) in enumerate(opcode["_nb_ops"]): - fobj.write(DEFINE.format(op, i)) - - fobj.write(opcode_h_footer) - with open(internal_opcode_h, 'w') as iobj: iobj.write(internal_header) @@ -91,8 +60,8 @@ def main(opcode_py, iobj.write(internal_footer) - print(f"{opcode_h} regenerated from {opcode_py}") + print(f"{internal_opcode_h} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3]) + main(sys.argv[1], sys.argv[2]) From 80f30cf51bd89411ef1220d714b33667d6a39901 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 17 Aug 2023 19:19:07 +0300 Subject: [PATCH 179/598] gh-102029: Deprecate passing arguments to `_PyRLock` in `threading` (#102071) --- Doc/whatsnew/3.13.rst | 6 +++++ Lib/test/test_threading.py | 24 +++++++++++++++++++ Lib/threading.py | 7 ++++++ ...-02-20-15-41-59.gh-issue-102029.9ZPG99.rst | 1 + 4 files changed, 38 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 519dee5eb7d6cf..13ae6e595dc993 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -357,6 +357,12 @@ Pending Removal in Python 3.15 They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) +* Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + Pending Removal in Python 3.16 ------------------------------ diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4a91eef31c4b8b..6465a446565844 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1748,6 +1748,30 @@ class PyRLockTests(lock_tests.RLockTests): class CRLockTests(lock_tests.RLockTests): locktype = staticmethod(threading._CRLock) + def test_signature(self): # gh-102029 + with warnings.catch_warnings(record=True) as warnings_log: + threading.RLock() + self.assertEqual(warnings_log, []) + + arg_types = [ + ((1,), {}), + ((), {'a': 1}), + ((1, 2), {'a': 1}), + ] + for args, kwargs in arg_types: + with self.subTest(args=args, kwargs=kwargs): + with self.assertWarns(DeprecationWarning): + threading.RLock(*args, **kwargs) + + # Subtypes with custom `__init__` are allowed (but, not recommended): + class CustomRLock(self.locktype): + def __init__(self, a, *, b) -> None: + super().__init__() + + with warnings.catch_warnings(record=True) as warnings_log: + CustomRLock(1, b=2) + self.assertEqual(warnings_log, []) + class EventTests(lock_tests.EventTests): eventtype = staticmethod(threading.Event) diff --git a/Lib/threading.py b/Lib/threading.py index e036cb891e196d..f6bbdb0d0c80ee 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -4,6 +4,7 @@ import sys as _sys import _thread import functools +import warnings from time import monotonic as _time from _weakrefset import WeakSet @@ -116,6 +117,12 @@ def RLock(*args, **kwargs): acquired it. """ + if args or kwargs: + warnings.warn( + 'Passing arguments to RLock is deprecated and will be removed in 3.15', + DeprecationWarning, + stacklevel=2, + ) if _CRLock is None: return _PyRLock(*args, **kwargs) return _CRLock(*args, **kwargs) diff --git a/Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst b/Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst new file mode 100644 index 00000000000000..4d6f05f2524528 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst @@ -0,0 +1 @@ +Deprecate passing any arguments to :func:`threading.RLock`. From 292a22bdc22f2aa70c96e9e53ca6d6b0c5f8d5bf Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 17 Aug 2023 20:16:08 +0200 Subject: [PATCH 180/598] gh-104683: Argument Clinic: Remove unreachable code from _module_and_class() (#108092) 'not hasattr(parent, "classes")' is always false, since 'parent' is an instance of either the Module, Class, or Clinic classes, and all of them has a "classes" attribute. --- Tools/clinic/clinic.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 9f7c47430772f7..1593dc49e07e1f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2427,8 +2427,6 @@ def _module_and_class( if child: parent = module = child continue - if not hasattr(parent, 'classes'): - return module, cls child = parent.classes.get(field) if not child: fullname = ".".join(so_far) From 61c7249759ce88465ea655d5c19d17d03ff3f74b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 17 Aug 2023 11:29:58 -0700 Subject: [PATCH 181/598] gh-106581: Project through calls (#108067) This finishes the work begun in gh-107760. When, while projecting a superblock, we encounter a call to a short, simple function, the superblock will now enter the function using `_PUSH_FRAME`, continue through it, and leave it using `_POP_FRAME`, and then continue through the original code. Multiple frame pushes and pops are even possible. It is also possible to stop appending to the superblock in the middle of a called function, when running out of space or encountering an unsupported bytecode. --- Include/internal/pycore_ceval.h | 1 + Include/internal/pycore_function.h | 9 ++ Include/internal/pycore_opcode_metadata.h | 87 +++++++++-------- Lib/test/test_capi/test_misc.py | 1 + Lib/test/test_code.py | 2 +- Objects/codeobject.c | 3 + Objects/funcobject.c | 79 +++++++++++++++- Python/abstract_interp_cases.c.h | 9 ++ Python/bytecodes.c | 56 ++++++----- Python/ceval.c | 14 +-- Python/ceval_macros.h | 4 + Python/executor_cases.c.h | 46 ++++++++- Python/generated_cases.c.h | 109 ++++++++++++++++------ Python/optimizer.c | 90 +++++++++++++++++- Tools/cases_generator/analysis.py | 6 +- Tools/cases_generator/stacking.py | 2 +- 16 files changed, 409 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 05b7380597812b..0e3a99be8c36aa 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -171,6 +171,7 @@ void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject * PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); #ifdef __cplusplus diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index e844d323ec7927..3f3da8a44b77e4 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -16,13 +16,22 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 +#define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct _py_func_state { uint32_t next_version; + // Borrowed references to function objects whose + // func_version % FUNC_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the function is deallocated. + PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); + extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index afe8aa172b703b..396d194ed2734e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -33,35 +33,36 @@ #define _BINARY_OP_SUBTRACT_FLOAT 309 #define _GUARD_BOTH_UNICODE 310 #define _BINARY_OP_ADD_UNICODE 311 -#define _LOAD_LOCALS 312 -#define _LOAD_FROM_DICT_OR_GLOBALS 313 -#define _GUARD_GLOBALS_VERSION 314 -#define _GUARD_BUILTINS_VERSION 315 -#define _LOAD_GLOBAL_MODULE 316 -#define _LOAD_GLOBAL_BUILTINS 317 -#define _GUARD_TYPE_VERSION 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _LOAD_ATTR_INSTANCE_VALUE 320 -#define IS_NONE 321 -#define _ITER_CHECK_LIST 322 -#define _IS_ITER_EXHAUSTED_LIST 323 -#define _ITER_NEXT_LIST 324 -#define _ITER_CHECK_TUPLE 325 -#define _IS_ITER_EXHAUSTED_TUPLE 326 -#define _ITER_NEXT_TUPLE 327 -#define _ITER_CHECK_RANGE 328 -#define _IS_ITER_EXHAUSTED_RANGE 329 -#define _ITER_NEXT_RANGE 330 -#define _CHECK_PEP_523 331 -#define _CHECK_FUNCTION_EXACT_ARGS 332 -#define _CHECK_STACK_SPACE 333 -#define _INIT_CALL_PY_EXACT_ARGS 334 -#define _PUSH_FRAME 335 -#define _POP_JUMP_IF_FALSE 336 -#define _POP_JUMP_IF_TRUE 337 -#define JUMP_TO_TOP 338 -#define SAVE_CURRENT_IP 339 -#define INSERT 340 +#define _POP_FRAME 312 +#define _LOAD_LOCALS 313 +#define _LOAD_FROM_DICT_OR_GLOBALS 314 +#define _GUARD_GLOBALS_VERSION 315 +#define _GUARD_BUILTINS_VERSION 316 +#define _LOAD_GLOBAL_MODULE 317 +#define _LOAD_GLOBAL_BUILTINS 318 +#define _GUARD_TYPE_VERSION 319 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 +#define _LOAD_ATTR_INSTANCE_VALUE 321 +#define IS_NONE 322 +#define _ITER_CHECK_LIST 323 +#define _IS_ITER_EXHAUSTED_LIST 324 +#define _ITER_NEXT_LIST 325 +#define _ITER_CHECK_TUPLE 326 +#define _IS_ITER_EXHAUSTED_TUPLE 327 +#define _ITER_NEXT_TUPLE 328 +#define _ITER_CHECK_RANGE 329 +#define _IS_ITER_EXHAUSTED_RANGE 330 +#define _ITER_NEXT_RANGE 331 +#define _CHECK_PEP_523 332 +#define _CHECK_FUNCTION_EXACT_ARGS 333 +#define _CHECK_STACK_SPACE 334 +#define _INIT_CALL_PY_EXACT_ARGS 335 +#define _PUSH_FRAME 336 +#define _POP_JUMP_IF_FALSE 337 +#define _POP_JUMP_IF_TRUE 338 +#define JUMP_TO_TOP 339 +#define SAVE_CURRENT_IP 340 +#define INSERT 341 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -197,6 +198,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg; case INTERPRETER_EXIT: return 1; + case _POP_FRAME: + return 1; case RETURN_VALUE: return 1; case INSTRUMENTED_RETURN_VALUE: @@ -723,6 +726,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case INTERPRETER_EXIT: return 0; + case _POP_FRAME: + return 0; case RETURN_VALUE: return 0; case INSTRUMENTED_RETURN_VALUE: @@ -1191,7 +1196,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, - [END_FOR] = { true, INSTR_FMT_IB, 0 }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, 0 }, [END_SEND] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, 0 }, @@ -1205,14 +1210,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, 0 }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, 0 }, [UNARY_INVERT] = { true, INSTR_FMT_IX, 0 }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, @@ -1259,7 +1264,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IB, 0 }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, 0 }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, @@ -1400,6 +1405,7 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACR #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, + [RESUME] = { .nuops = 1, .uops = { { RESUME, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, @@ -1444,6 +1450,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 3, .uops = { { SAVE_IP, 7, 0 }, { SAVE_CURRENT_IP, 0, 0 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 4, .uops = { { LOAD_CONST, 0, 0 }, { SAVE_IP, 7, 0 }, { SAVE_CURRENT_IP, 0, 0 }, { _POP_FRAME, 0, 0 } } }, [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, @@ -1545,6 +1553,7 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_POP_FRAME] = "_POP_FRAME", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3dfbfdc26e7416..18a0476122dabf 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2633,6 +2633,7 @@ def dummy(x): self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index ca06a39f5df142..e056c16466e8c4 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -264,7 +264,7 @@ def func2(): ("co_posonlyargcount", 0), ("co_kwonlyargcount", 0), ("co_nlocals", 1), - ("co_stacksize", 0), + ("co_stacksize", 1), ("co_flags", code.co_flags | inspect.CO_COROUTINE), ("co_firstlineno", 100), ("co_code", code2.co_code), diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 2c9c8cec77ff9f..4d6efe938f45d6 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -396,6 +396,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) int nlocals, ncellvars, nfreevars; get_localsplus_counts(con->localsplusnames, con->localspluskinds, &nlocals, &ncellvars, &nfreevars); + if (con->stacksize == 0) { + con->stacksize = 1; + } co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 8c0bface3ac710..33191d23f18230 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -223,7 +223,73 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname return NULL; } -uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) +/* +Function versions +----------------- + +Function versions are used to detect when a function object has been +updated, invalidating inline cache data used by the `CALL` bytecode +(notably `CALL_PY_EXACT_ARGS` and a few other `CALL` specializations). + +They are also used by the Tier 2 superblock creation code to find +the function being called (and from there the code object). + +How does a function's `func_version` field get initialized? + +- `PyFunction_New` and friends initialize it to 0. +- The `MAKE_FUNCTION` instruction sets it from the code's `co_version`. +- It is reset to 0 when various attributes like `__code__` are set. +- A new version is allocated by `_PyFunction_GetVersionForCurrentState` + when the specializer needs a version and the version is 0. + +The latter allocates versions using a counter in the interpreter state; +when the counter wraps around to 0, no more versions are allocated. +There is one other special case: functions with a non-standard +`vectorcall` field are not given a version. + +When the function version is 0, the `CALL` bytecode is not specialized. + +Code object versions +-------------------- + +So where to code objects get their `co_version`? There is a single +static global counter, `_Py_next_func_version`. This is initialized in +the generated (!) file `Python/deepfreeze/deepfreeze.c`, to 1 plus the +number of deep-frozen function objects in that file. +(In `_bootstrap_python.c` and `freeze_module.c` it is initialized to 1.) + +Code objects get a new `co_version` allocated from this counter upon +creation. Since code objects are nominally immutable, `co_version` can +not be invalidated. The only way it can be 0 is when 2**32 or more +code objects have been created during the process's lifetime. +(The counter isn't reset by `fork()`, extending the lifetime.) +*/ + +void +_PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) +{ + func->func_version = version; + if (version != 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + interp->func_state.func_version_cache[ + version % FUNC_VERSION_CACHE_SIZE] = func; + } +} + +PyFunctionObject * +_PyFunction_LookupByVersion(uint32_t version) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyFunctionObject *func = interp->func_state.func_version_cache[ + version % FUNC_VERSION_CACHE_SIZE]; + if (func != NULL && func->func_version == version) { + return (PyFunctionObject *)Py_NewRef(func); + } + return NULL; +} + +uint32_t +_PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { if (func->func_version != 0) { return func->func_version; @@ -236,7 +302,7 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) return 0; } uint32_t v = interp->func_state.next_version++; - func->func_version = v; + _PyFunction_SetVersion(func, v); return v; } @@ -851,6 +917,15 @@ func_dealloc(PyFunctionObject *op) if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } + if (op->func_version != 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyFunctionObject **slot = + interp->func_state.func_version_cache + + (op->func_version % FUNC_VERSION_CACHE_SIZE); + if (*slot == op) { + *slot = NULL; + } + } (void)func_clear(op); // These aren't cleared by func_clear(). Py_DECREF(op->func_code); diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index eef071119bcd84..1b99b929fa8014 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -7,6 +7,10 @@ break; } + case RESUME: { + break; + } + case POP_TOP: { STACK_SHRINK(1); break; @@ -191,6 +195,11 @@ break; } + case _POP_FRAME: { + STACK_SHRINK(1); + break; + } + case GET_AITER: { PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6f17472e04e5e3..ae459cabaddc05 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -133,6 +133,7 @@ dummy_func( } inst(RESUME, (--)) { + #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { @@ -140,7 +141,9 @@ dummy_func( ERROR_IF(err, error); next_instr--; } - else if (oparg < 2) { + else + #endif + if (oparg < 2) { CHECK_EVAL_BREAKER(); } } @@ -757,21 +760,37 @@ dummy_func( return retval; } - inst(RETURN_VALUE, (retval --)) { - STACK_SHRINK(1); + // The stack effect here is ambiguous. + // We definitely pop the return value off the stack on entry. + // We also push it onto the stack on exit, but that's a + // different frame, and it's accounted for by _PUSH_FRAME. + op(_POP_FRAME, (retval --)) { assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + #if TIER_ONE goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif } + macro(RETURN_VALUE) = + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _POP_FRAME; + inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -785,27 +804,17 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; } - inst(RETURN_CONST, (--)) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } + macro(RETURN_CONST) = + LOAD_CONST + + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _POP_FRAME; inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); @@ -821,7 +830,7 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -3545,7 +3554,8 @@ dummy_func( goto error; } - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; } diff --git a/Python/ceval.c b/Python/ceval.c index 1e2262c1f18c3e..329a1a17cf09d4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -222,8 +222,6 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); -static void -_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); #ifdef HAVE_ERRNO_H #include @@ -603,10 +601,6 @@ int _Py_CheckRecursiveCallPy( } -static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { - tstate->py_recursion_remaining++; -} - static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ @@ -731,7 +725,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // When tracing executed uops, also trace bytecode char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = (*uop_debug - '0') >= 4; // TODO: Parse an int and all that + lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that } } } @@ -918,7 +912,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame == &entry_frame) { /* Restore previous frame and exit */ @@ -1487,8 +1481,8 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) frame->previous = NULL; } -static void -_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) +void +_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { if (frame->owner == FRAME_OWNED_BY_THREAD) { clear_thread_frame(tstate, frame); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 08f19cd9a397f1..635b8e501e523e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -369,3 +369,7 @@ static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { return (tstate->py_recursion_remaining-- <= 0) && _Py_CheckRecursiveCallPy(tstate); } + +static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { + tstate->py_recursion_remaining++; +} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9fbf026f164a60..89a5bbfecded0e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7,6 +7,23 @@ break; } + case RESUME: { + #if TIER_ONE + assert(frame == tstate->current_frame); + /* Possibly combine this with eval breaker */ + if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr--; + } + else + #endif + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } + break; + } + case LOAD_FAST_CHECK: { PyObject *value; value = GETLOCAL(oparg); @@ -666,6 +683,32 @@ break; } + case _POP_FRAME: { + PyObject *retval; + retval = stack_pointer[-1]; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + #if TIER_ONE + goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + break; + } + case GET_AITER: { PyObject *obj; PyObject *iter; @@ -2607,7 +2650,8 @@ goto error; } - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 80af8a7bcd56df..f6322df566c650 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,7 @@ } TARGET(RESUME) { + #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { @@ -15,7 +16,9 @@ if (err) goto error; next_instr--; } - else if (oparg < 2) { + else + #endif + if (oparg < 2) { CHECK_EVAL_BREAKER(); } DISPATCH(); @@ -970,20 +973,40 @@ TARGET(RETURN_VALUE) { PyObject *retval; + // SAVE_CURRENT_IP + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _POP_FRAME retval = stack_pointer[-1]; STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - STACK_SHRINK(1); + { + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + #if TIER_ONE + goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(INSTRUMENTED_RETURN_VALUE) { @@ -1001,7 +1024,7 @@ // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -1009,19 +1032,46 @@ } TARGET(RETURN_CONST) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + PyObject *value; + PyObject *retval; + // LOAD_CONST + { + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + } + // SAVE_CURRENT_IP + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _POP_FRAME + retval = value; + { + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + #if TIER_ONE + goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(INSTRUMENTED_RETURN_CONST) { @@ -1038,7 +1088,7 @@ // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -4575,7 +4625,8 @@ goto error; } - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; DISPATCH(); diff --git a/Python/optimizer.c b/Python/optimizer.c index 559c4ae987263e..57518404c3f19d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -373,6 +373,8 @@ static PyTypeObject UOpExecutor_Type = { .tp_as_sequence = &uop_as_sequence, }; +#define TRACE_STACK_SIZE 5 + static int translate_bytecode_to_trace( PyCodeObject *code, @@ -380,10 +382,16 @@ translate_bytecode_to_trace( _PyUOpInstruction *trace, int buffer_size) { + PyCodeObject *initial_code = code; _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; int max_length = buffer_size; int reserved = 0; + struct { + PyCodeObject *code; + _Py_CODEUNIT *instr; + } trace_stack[TRACE_STACK_SIZE]; + int trace_stack_depth = 0; #ifdef Py_DEBUG char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); @@ -441,6 +449,24 @@ translate_bytecode_to_trace( // Reserve space for main+stub uops, plus 2 for SAVE_IP and EXIT_TRACE #define RESERVE(main, stub) RESERVE_RAW((main) + (stub) + 2, uop_name(opcode)) +// Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) +#define TRACE_STACK_PUSH() \ + if (trace_stack_depth >= TRACE_STACK_SIZE) { \ + DPRINTF(2, "Trace stack overflow\n"); \ + ADD_TO_TRACE(SAVE_IP, 0, 0); \ + goto done; \ + } \ + trace_stack[trace_stack_depth].code = code; \ + trace_stack[trace_stack_depth].instr = instr; \ + trace_stack_depth++; +#define TRACE_STACK_POP() \ + if (trace_stack_depth <= 0) { \ + Py_FatalError("Trace stack underflow\n"); \ + } \ + trace_stack_depth--; \ + code = trace_stack[trace_stack_depth].code; \ + instr = trace_stack[trace_stack_depth].instr; + DPRINTF(4, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -448,6 +474,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); +top: // Jump here after _PUSH_FRAME for (;;) { RESERVE_RAW(2, "epilogue"); // Always need space for SAVE_IP and EXIT_TRACE ADD_TO_TRACE(SAVE_IP, INSTR_IP(instr, code), 0); @@ -508,7 +535,7 @@ translate_bytecode_to_trace( case JUMP_BACKWARD: { - if (instr + 2 - oparg == initial_instr) { + if (instr + 2 - oparg == initial_instr && code == initial_code) { RESERVE(1, 0); ADD_TO_TRACE(JUMP_TO_TOP, 0, 0); } @@ -573,6 +600,14 @@ translate_bytecode_to_trace( // Reserve space for nuops (+ SAVE_IP + EXIT_TRACE) int nuops = expansion->nuops; RESERVE(nuops, 0); + if (expansion->uops[nuops-1].uop == _POP_FRAME) { + // Check for trace stack underflow now: + // We can't bail e.g. in the middle of + // LOAD_CONST + _POP_FRAME. + if (trace_stack_depth == 0) { + DPRINTF(2, "Trace stack underflow\n"); + goto done;} + } uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM for (int i = 0; i < nuops; i++) { oparg = orig_oparg; @@ -619,8 +654,57 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } ADD_TO_TRACE(expansion->uops[i].uop, oparg, operand); + if (expansion->uops[i].uop == _POP_FRAME) { + TRACE_STACK_POP(); + DPRINTF(2, + "Returning to %s (%s:%d) at byte offset %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(instr, code)); + goto top; + } if (expansion->uops[i].uop == _PUSH_FRAME) { assert(i + 1 == nuops); + int func_version_offset = + offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) + // Add one to account for the actual opcode/oparg pair: + + 1; + uint32_t func_version = read_u32(&instr[func_version_offset].cache); + PyFunctionObject *func = _PyFunction_LookupByVersion(func_version); + DPRINTF(3, "Function object: %p\n", func); + if (func != NULL) { + PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(func); + if (new_code == code) { + // Recursive call, bail (we could be here forever). + DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", + PyUnicode_AsUTF8(new_code->co_qualname), + PyUnicode_AsUTF8(new_code->co_filename), + new_code->co_firstlineno); + ADD_TO_TRACE(SAVE_IP, 0, 0); + goto done; + } + if (new_code->co_version != func_version) { + // func.__code__ was updated. + // Perhaps it may happen again, so don't bother tracing. + // TODO: Reason about this -- is it better to bail or not? + DPRINTF(2, "Bailing because co_version != func_version\n"); + ADD_TO_TRACE(SAVE_IP, 0, 0); + goto done; + } + // Increment IP to the return address + instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; + TRACE_STACK_PUSH(); + code = new_code; + instr = _PyCode_CODE(code); + DPRINTF(2, + "Continuing in %s (%s:%d) at byte offset %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(instr, code)); + goto top; + } ADD_TO_TRACE(SAVE_IP, 0, 0); goto done; } @@ -639,6 +723,10 @@ translate_bytecode_to_trace( } // End for (;;) done: + while (trace_stack_depth > 0) { + TRACE_STACK_POP(); + } + assert(code == initial_code); // Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE if (trace_length > 3) { ADD_TO_TRACE(EXIT_TRACE, 0, 0); diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 2db1cd01c19ae5..48f2db981c95b6 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -364,10 +364,12 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: case Instruction() as instr: part, offset = self.analyze_instruction(instr, offset) parts.append(part) - flags.add(instr.instr_flags) + if instr.name != "SAVE_IP": + # SAVE_IP in a macro is a no-op in Tier 1 + flags.add(instr.instr_flags) case _: typing.assert_never(component) - format = "IB" + format = "IB" if flags.HAS_ARG_FLAG else "IX" if offset: format += "C" + "0" * (offset - 1) return MacroInstruction(macro.name, format, flags, macro, parts, offset) diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 8361eb99f88a7c..632298a567dd40 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -380,7 +380,7 @@ def write_components( poke.as_stack_effect(lax=True), ) - if mgr.instr.name == "_PUSH_FRAME": + if mgr.instr.name in ("_PUSH_FRAME", "_POP_FRAME"): # Adjust stack to min_offset (input effects materialized) out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) # Use clone() since adjust_inverse() mutates final_offset From 75b3db8445188c2ad38cabc0021af694df0829b8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 17 Aug 2023 19:39:42 +0100 Subject: [PATCH 182/598] gh-107944: Improve error message for function calls with bad keyword arguments (#107969) --- Include/internal/pycore_pyerrors.h | 2 +- Lib/test/test_call.py | 68 +++++++++++++++++++ ...-08-15-11-09-50.gh-issue-107944.zQLp3j.rst | 2 + Python/ceval.c | 31 ++++++++- Python/suggestions.c | 14 ++-- 5 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 45929f40a05496..91fd68984523dd 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -150,7 +150,7 @@ extern PyObject* _PyExc_PrepReraiseStar( extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); - +extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index c3c3b1853b5736..008a8c1f0cb876 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -915,6 +915,74 @@ def test_multiple_values(self): with self.check_raises_type_error(msg): A().method_two_args("x", "y", x="oops") +@cpython_only +class TestErrorMessagesSuggestions(unittest.TestCase): + @contextlib.contextmanager + def check_suggestion_includes(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) + + @contextlib.contextmanager + def check_suggestion_not_pressent(self): + with self.assertRaises(TypeError) as cm: + yield + self.assertNotIn("Did you mean", str(cm.exception)) + + def test_unexpected_keyword_suggestion_valid_positions(self): + def foo(blech=None, /, aaa=None, *args, late1=None): + pass + + cases = [ + ("blach", None), + ("aa", "aaa"), + ("orgs", None), + ("late11", "late1"), + ] + + for keyword, suggestion in cases: + with self.subTest(keyword): + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_pressent() + with ctx: + foo(**{keyword:None}) + + def test_unexpected_keyword_suggestion_kinds(self): + + def substitution(noise=None, more_noise=None, a = None, blech = None): + pass + + def elimination(noise = None, more_noise = None, a = None, blch = None): + pass + + def addition(noise = None, more_noise = None, a = None, bluchin = None): + pass + + def substitution_over_elimination(blach = None, bluc = None): + pass + + def substitution_over_addition(blach = None, bluchi = None): + pass + + def elimination_over_addition(bluc = None, blucha = None): + pass + + def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): + pass + + for func, suggestion in [ + (addition, "bluchin"), + (substitution, "blech"), + (elimination, "blch"), + (addition, "bluchin"), + (substitution_over_elimination, "blach"), + (substitution_over_addition, "blach"), + (elimination_over_addition, "bluc"), + (case_change_over_substitution, "BLuch"), + ]: + with self.subTest(suggestion): + with self.check_suggestion_includes(suggestion): + func(bluch=None) + @cpython_only class TestRecursion(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst new file mode 100644 index 00000000000000..9a53332c70261d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst @@ -0,0 +1,2 @@ +Improve error message for function calls with bad keyword arguments. Patch +by Pablo Galindo diff --git a/Python/ceval.c b/Python/ceval.c index 329a1a17cf09d4..f7dfaebcdd8f84 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -26,6 +26,7 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uops.h" // _PyUOpExecutorObject +#include "pycore_pyerrors.h" #include "pycore_dict.h" #include "dictobject.h" @@ -1337,9 +1338,33 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto kw_fail; } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'", - func->func_qualname, keyword); + PyObject* suggestion_keyword = NULL; + if (total_args > co->co_posonlyargcount) { + PyObject* possible_keywords = PyList_New(total_args - co->co_posonlyargcount); + + if (!possible_keywords) { + PyErr_Clear(); + } else { + for (Py_ssize_t k = co->co_posonlyargcount; k < total_args; k++) { + PyList_SET_ITEM(possible_keywords, k - co->co_posonlyargcount, co_varnames[k]); + } + + suggestion_keyword = _Py_CalculateSuggestions(possible_keywords, keyword); + Py_DECREF(possible_keywords); + } + } + + if (suggestion_keyword) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", + func->func_qualname, keyword, suggestion_keyword); + Py_DECREF(suggestion_keyword); + } else { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'", + func->func_qualname, keyword); + } + goto kw_fail; } diff --git a/Python/suggestions.c b/Python/suggestions.c index 47aeb08180f6b1..12097f793e3575 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -126,8 +126,8 @@ levenshtein_distance(const char *a, size_t a_size, return result; } -static inline PyObject * -calculate_suggestions(PyObject *dir, +PyObject * +_Py_CalculateSuggestions(PyObject *dir, PyObject *name) { assert(!PyErr_Occurred()); @@ -195,7 +195,7 @@ get_suggestions_for_attribute_error(PyAttributeErrorObject *exc) return NULL; } - PyObject *suggestions = calculate_suggestions(dir, name); + PyObject *suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); return suggestions; } @@ -259,7 +259,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) } } - PyObject *suggestions = calculate_suggestions(dir, name); + PyObject *suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); if (suggestions != NULL || PyErr_Occurred()) { return suggestions; @@ -269,7 +269,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) if (dir == NULL) { return NULL; } - suggestions = calculate_suggestions(dir, name); + suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); if (suggestions != NULL || PyErr_Occurred()) { return suggestions; @@ -279,7 +279,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) if (dir == NULL) { return NULL; } - suggestions = calculate_suggestions(dir, name); + suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); return suggestions; @@ -371,7 +371,7 @@ offer_suggestions_for_import_error(PyImportErrorObject *exc) return NULL; } - PyObject *suggestion = calculate_suggestions(dir, name); + PyObject *suggestion = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); if (!suggestion) { return NULL; From 5c76899dadf3bdcfdedf6f30b3ab9742cb87af04 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 17 Aug 2023 21:19:01 +0200 Subject: [PATCH 183/598] Docs: Fix Sphinx warnings in io.rst (#107903) - Mark up parameter and argument names properly - If possible, link to docs for methods like `seek`, `tell`, `write`, `read`, etc. Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/io.rst | 71 ++++++++++++++++++++++---------------------- Doc/tools/.nitignore | 1 - 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 7eec1f87583b87..66273d9ed1ff0a 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,9 +38,9 @@ location), or only sequential access (for example in the case of a socket or pipe). All streams are careful about the type of data you give to them. For example -giving a :class:`str` object to the ``write()`` method of a binary stream +giving a :class:`str` object to the :meth:`!write` method of a binary stream will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the -``write()`` method of a text stream. +:meth:`!write` method of a text stream. .. versionchanged:: 3.3 Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since @@ -146,7 +146,7 @@ Opt-in EncodingWarning See :pep:`597` for more details. To find where the default locale encoding is used, you can enable -the ``-X warn_default_encoding`` command line option or set the +the :option:`-X warn_default_encoding <-X>` command line option or set the :envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will emit an :exc:`EncodingWarning` when the default encoding is used. @@ -175,7 +175,7 @@ High-level Module Interface .. audit-event:: open path,mode,flags io.open This function raises an :ref:`auditing event ` ``open`` with - arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments *path*, *mode* and *flags*. The *mode* and *flags* arguments may have been modified or inferred from the original call. @@ -184,10 +184,10 @@ High-level Module Interface Opens the provided file with mode ``'rb'``. This function should be used when the intent is to treat the contents as executable code. - ``path`` should be a :class:`str` and an absolute path. + *path* should be a :class:`str` and an absolute path. The behavior of this function may be overridden by an earlier call to the - :c:func:`PyFile_SetOpenCodeHook`. However, assuming that ``path`` is a + :c:func:`PyFile_SetOpenCodeHook`. However, assuming that *path* is a :class:`str` and an absolute path, ``open_code(path)`` should always behave the same as ``open(path, 'rb')``. Overriding the behavior is intended for additional validation or preprocessing of the file. @@ -258,7 +258,7 @@ standard stream implementations. The abstract base classes also provide default implementations of some methods in order to help implementation of concrete stream classes. For example, :class:`BufferedIOBase` provides unoptimized implementations of - :meth:`~IOBase.readinto` and :meth:`~IOBase.readline`. + :meth:`!readinto` and :meth:`!readline`. At the top of the I/O hierarchy is the abstract base class :class:`IOBase`. It defines the basic interface to a stream. Note, however, that there is no @@ -320,8 +320,8 @@ I/O Base Classes implementations represent a file that cannot be read, written or seeked. - Even though :class:`IOBase` does not declare :meth:`read` - or :meth:`write` because their signatures will vary, implementations and + Even though :class:`IOBase` does not declare :meth:`!read` + or :meth:`!write` because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`) when operations they do not support are called. @@ -379,8 +379,8 @@ I/O Base Classes .. method:: readable() - Return ``True`` if the stream can be read from. If ``False``, :meth:`read` - will raise :exc:`OSError`. + Return ``True`` if the stream can be read from. + If ``False``, :meth:`!read` will raise :exc:`OSError`. .. method:: readline(size=-1, /) @@ -401,25 +401,25 @@ I/O Base Classes hint. Note that it's already possible to iterate on file objects using ``for - line in file: ...`` without calling ``file.readlines()``. + line in file: ...`` without calling :meth:`!file.readlines`. .. method:: seek(offset, whence=SEEK_SET, /) Change the stream position to the given byte *offset*. *offset* is interpreted relative to the position indicated by *whence*. The default - value for *whence* is :data:`SEEK_SET`. Values for *whence* are: + value for *whence* is :data:`!SEEK_SET`. Values for *whence* are: - * :data:`SEEK_SET` or ``0`` -- start of the stream (the default); + * :data:`!SEEK_SET` or ``0`` -- start of the stream (the default); *offset* should be zero or positive - * :data:`SEEK_CUR` or ``1`` -- current stream position; *offset* may + * :data:`!SEEK_CUR` or ``1`` -- current stream position; *offset* may be negative - * :data:`SEEK_END` or ``2`` -- end of the stream; *offset* is usually + * :data:`!SEEK_END` or ``2`` -- end of the stream; *offset* is usually negative Return the new absolute position. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. versionadded:: 3.3 Some operating systems could support additional values, like @@ -450,7 +450,7 @@ I/O Base Classes .. method:: writable() Return ``True`` if the stream supports writing. If ``False``, - :meth:`write` and :meth:`truncate` will raise :exc:`OSError`. + :meth:`!write` and :meth:`truncate` will raise :exc:`OSError`. .. method:: writelines(lines, /) @@ -654,8 +654,9 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`read` (when called with a positive argument), :meth:`readinto` - and :meth:`write` methods on this class will only make one system call. + The :meth:`~RawIOBase.read` (when called with a positive argument), + :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this + class will only make one system call. A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with @@ -791,8 +792,8 @@ than raw I/O does. object under various conditions, including: * when the buffer gets too small for all pending data; - * when :meth:`flush()` is called; - * when a :meth:`seek()` is requested (for :class:`BufferedRandom` objects); + * when :meth:`flush` is called; + * when a :meth:`~IOBase.seek` is requested (for :class:`BufferedRandom` objects); * when the :class:`BufferedWriter` object is closed or destroyed. The constructor creates a :class:`BufferedWriter` for the given writeable @@ -826,8 +827,8 @@ than raw I/O does. :data:`DEFAULT_BUFFER_SIZE`. :class:`BufferedRandom` is capable of anything :class:`BufferedReader` or - :class:`BufferedWriter` can do. In addition, :meth:`seek` and :meth:`tell` - are guaranteed to be implemented. + :class:`BufferedWriter` can do. In addition, :meth:`~IOBase.seek` and + :meth:`~IOBase.tell` are guaranteed to be implemented. .. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE, /) @@ -904,7 +905,7 @@ Text I/O .. method:: readline(size=-1, /) - Read until newline or EOF and return a single ``str``. If the stream is + Read until newline or EOF and return a single :class:`str`. If the stream is already at EOF, an empty string is returned. If *size* is specified, at most *size* characters will be read. @@ -913,22 +914,22 @@ Text I/O Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is - :data:`SEEK_SET`. + :data:`!SEEK_SET`. - * :data:`SEEK_SET` or ``0``: seek from the start of the stream + * :data:`!SEEK_SET` or ``0``: seek from the start of the stream (the default); *offset* must either be a number returned by :meth:`TextIOBase.tell`, or zero. Any other *offset* value produces undefined behaviour. - * :data:`SEEK_CUR` or ``1``: "seek" to the current position; + * :data:`!SEEK_CUR` or ``1``: "seek" to the current position; *offset* must be zero, which is a no-operation (all other values are unsupported). - * :data:`SEEK_END` or ``2``: seek to the end of the stream; + * :data:`!SEEK_END` or ``2``: seek to the end of the stream; *offset* must be zero (all other values are unsupported). Return the new absolute position as an opaque number. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. method:: tell() @@ -988,10 +989,10 @@ Text I/O takes place. If *newline* is any of the other legal values, any ``'\n'`` characters written are translated to the given string. - If *line_buffering* is ``True``, :meth:`flush` is implied when a call to + If *line_buffering* is ``True``, :meth:`~IOBase.flush` is implied when a call to write contains a newline character or a carriage return. - If *write_through* is ``True``, calls to :meth:`write` are guaranteed + If *write_through* is ``True``, calls to :meth:`~BufferedIOBase.write` are guaranteed not to be buffered: any data written on the :class:`TextIOWrapper` object is immediately handled to its underlying binary *buffer*. @@ -1070,7 +1071,7 @@ Text I/O .. method:: getvalue() - Return a ``str`` containing the entire contents of the buffer. + Return a :class:`str` containing the entire contents of the buffer. Newlines are decoded as if by :meth:`~TextIOBase.read`, although the stream position is not changed. @@ -1125,7 +1126,7 @@ Text I/O over a binary storage (such as a file) is significantly slower than binary I/O over the same storage, because it requires conversions between unicode and binary data using a character codec. This can become noticeable handling huge amounts of text data like large log files. Also, -:meth:`TextIOWrapper.tell` and :meth:`TextIOWrapper.seek` are both quite slow +:meth:`~TextIOBase.tell` and :meth:`~TextIOBase.seek` are both quite slow due to the reconstruction algorithm used. :class:`StringIO`, however, is a native in-memory unicode container and will @@ -1135,7 +1136,7 @@ Multi-threading ^^^^^^^^^^^^^^^ :class:`FileIO` objects are thread-safe to the extent that the operating system -calls (such as ``read(2)`` under Unix) they wrap are thread-safe too. +calls (such as :manpage:`read(2)` under Unix) they wrap are thread-safe too. Binary buffered objects (instances of :class:`BufferedReader`, :class:`BufferedWriter`, :class:`BufferedRandom` and :class:`BufferedRWPair`) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0f68cefc92b97b..654287cc50b55b 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -89,7 +89,6 @@ Doc/library/http.server.rst Doc/library/importlib.resources.rst Doc/library/importlib.rst Doc/library/inspect.rst -Doc/library/io.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst From 02079b010c39a89b284e8f0bb6d5f378e554260e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 17 Aug 2023 22:41:35 +0200 Subject: [PATCH 184/598] gh-107801: Improve the docs of the SEEK_* constants (#108099) --- Doc/library/os.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 94b76e4052b94b..4668984b10c690 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1180,16 +1180,26 @@ as internal buffering of data. SEEK_CUR SEEK_END - Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on :term:`file-like objects `, + for whence to adjust the file position indicator. + + :const:`SEEK_SET` + Adjust the file position relative to the beginning of the file. + :const:`SEEK_CUR` + Adjust the file position relative to the current file position. + :const:`SEEK_END` + Adjust the file position relative to the end of the file. + + Their values are 0, 1, and 2, respectively. .. data:: SEEK_HOLE SEEK_DATA Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` - method on file objects, for seeking file data and holes on sparsely - allocated files. + method on :term:`file-like objects `, + for seeking file data and holes on sparsely allocated files. :data:`!SEEK_DATA` Adjust the file offset to the next location containing data, From cc58ec9724772a8d5c4a5c9a6525f9f96e994227 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 18 Aug 2023 12:16:22 +0200 Subject: [PATCH 185/598] Docs: emphasise warning and add accurate markups for sys.unraisablehook (#108105) --- Doc/library/sys.rst | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 33391d11ab392d..b6346bafbf625e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1811,35 +1811,39 @@ always available. The *unraisable* argument has the following attributes: - * *exc_type*: Exception type. - * *exc_value*: Exception value, can be ``None``. - * *exc_traceback*: Exception traceback, can be ``None``. - * *err_msg*: Error message, can be ``None``. - * *object*: Object causing the exception, can be ``None``. + * :attr:`!exc_type`: Exception type. + * :attr:`!exc_value`: Exception value, can be ``None``. + * :attr:`!exc_traceback`: Exception traceback, can be ``None``. + * :attr:`!err_msg`: Error message, can be ``None``. + * :attr:`!object`: Object causing the exception, can be ``None``. - The default hook formats *err_msg* and *object* as: + The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if *err_msg* is ``None``. + if :attr:`!err_msg` is ``None``. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. - Storing *exc_value* using a custom hook can create a reference cycle. It - should be cleared explicitly to break the reference cycle when the - exception is no longer needed. + .. seealso:: + + :func:`excepthook` which handles uncaught exceptions. + + .. warning:: - Storing *object* using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing *object* after the custom - hook completes to avoid resurrecting objects. + Storing :attr:`!exc_value` using a custom hook can create a reference cycle. + It should be cleared explicitly to break the reference cycle when the + exception is no longer needed. - See also :func:`excepthook` which handles uncaught exceptions. + Storing :attr:`!object` using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing :attr:`!object` after the custom + hook completes to avoid resurrecting objects. .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook Raise an auditing event ``sys.unraisablehook`` with arguments - ``hook``, ``unraisable`` when an exception that cannot be handled occurs. - The ``unraisable`` object is the same as what will be passed to the hook. - If no hook has been set, ``hook`` may be ``None``. + *hook*, *unraisable* when an exception that cannot be handled occurs. + The *unraisable* object is the same as what will be passed to the hook. + If no hook has been set, *hook* may be ``None``. .. versionadded:: 3.8 From 3ff5ef2ad3d89c3ccf4e07ac8fdd798267ae6c61 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Aug 2023 12:34:41 +0200 Subject: [PATCH 186/598] gh-108014: Add Py_IsFinalizing() function (#108032) Co-authored-by: Serhiy Storchaka --- Doc/c-api/init.rst | 14 +++++++++++--- Doc/library/sys.rst | 4 ++-- Doc/whatsnew/3.13.rst | 4 ++++ Include/cpython/pylifecycle.h | 2 ++ Include/internal/pycore_pylifecycle.h | 1 - .../2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst | 2 ++ Modules/_asynciomodule.c | 2 +- Python/pylifecycle.c | 2 +- Python/sysmodule.c | 2 +- 9 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dd53fe2bc95696..60f5c81cff572c 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -373,6 +373,14 @@ Initializing and finalizing the interpreter :c:func:`Py_Initialize` is called again. +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + .. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of @@ -852,7 +860,7 @@ code, or when embedding the Python interpreter: .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -898,7 +906,7 @@ with sub-interpreters: .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -1180,7 +1188,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b6346bafbf625e..da2435aa4bc23d 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1121,8 +1121,8 @@ always available. .. function:: is_finalizing() - Return :const:`True` if the Python interpreter is - :term:`shutting down `, :const:`False` otherwise. + Return :const:`True` if the main Python interpreter is + :term:`shutting down `. Return :const:`False` otherwise. .. versionadded:: 3.5 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 13ae6e595dc993..85aa81b4a98d2c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -832,6 +832,10 @@ New Features not needed. (Contributed by Victor Stinner in :gh:`106004`.) +* Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is + :term:`shutting down `. + (Contributed by Victor Stinner in :gh:`108014`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index d425a233f71000..11b280afa8435b 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -81,3 +81,5 @@ PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( typedef void (*atexit_datacallbackfunc)(void *); PyAPI_FUNC(int) PyUnstable_AtExit( PyInterpreterState *, atexit_datacallbackfunc, void *); + +PyAPI_FUNC(int) Py_IsFinalizing(void); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index b4d5b1f1239e1d..56abd57d2bd5cf 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -98,7 +98,6 @@ extern int _Py_FdIsInteractive(FILE *fp, PyObject *filename); extern const char* _Py_gitidentifier(void); extern const char* _Py_gitversion(void); -extern int _Py_IsFinalizing(void); PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp); /* Random */ diff --git a/Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst b/Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst new file mode 100644 index 00000000000000..fee3d5b941dc8d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst @@ -0,0 +1,2 @@ +Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is +:term:`shutting down `. Patch by Victor Stinner. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 39c803355ba95b..6266dc8e3555f8 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -529,7 +529,7 @@ future_init(FutureObj *fut, PyObject *loop) } if (is_true && !_Py_IsInterpreterFinalizing(_PyInterpreterState_GET())) { /* Only try to capture the traceback if the interpreter is not being - finalized. The original motivation to add a `_Py_IsFinalizing()` + finalized. The original motivation to add a `Py_IsFinalizing()` call was to prevent SIGSEGV when a Future is created in a __del__ method, which is called during the interpreter shutdown and the traceback module is already unloaded. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 90633960cb00ee..263ead39fa472a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -129,7 +129,7 @@ _PyRuntime_Finalize(void) } int -_Py_IsFinalizing(void) +Py_IsFinalizing(void) { return _PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f82901181f8866..54533603f1a7f8 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2095,7 +2095,7 @@ static PyObject * sys_is_finalizing_impl(PyObject *module) /*[clinic end generated code: output=735b5ff7962ab281 input=f0df747a039948a5]*/ { - return PyBool_FromLong(_Py_IsFinalizing()); + return PyBool_FromLong(Py_IsFinalizing()); } #ifdef Py_STATS From fd195092204aa7fc9f13c5c6d423bc723d0b3520 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 18 Aug 2023 13:39:12 +0200 Subject: [PATCH 187/598] gh-108083: Don't ignore exceptions in sqlite3.Connection.__init__() and .close() (#108084) - Add explanatory comments - Add return value to connection_close() for propagating errors - Always check the return value of connection_exec_stmt() - Assert pre/post state in remove_callbacks() - Don't log unraisable exceptions in case of interpreter shutdown - Make sure we're not initialized if reinit fails - Try to close the database even if ROLLBACK fails Co-authored-by: Victor Stinner Co-authored-by: Serhiy Storchaka --- ...-08-17-12-59-35.gh-issue-108083.9J7UcT.rst | 3 + Modules/_sqlite/connection.c | 105 ++++++++++++------ 2 files changed, 77 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst diff --git a/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst b/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst new file mode 100644 index 00000000000000..ff499ced73a309 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst @@ -0,0 +1,3 @@ +Fix bugs in the constructor of :mod:`sqlite3.Connection` and +:meth:`sqlite3.Connection.close` where exceptions could be leaked. Patch by +Erlend E. Aasland. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0819acd0940964..282855f8865940 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -149,7 +149,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); static void free_callback_context(callback_context *ctx); static void set_callback_context(callback_context **ctx_pp, callback_context *ctx); -static void connection_close(pysqlite_Connection *self); +static int connection_close(pysqlite_Connection *self); PyObject *_pysqlite_query_execute(pysqlite_Cursor *, int, PyObject *, PyObject *); static PyObject * @@ -247,10 +247,13 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, } if (self->initialized) { + self->initialized = 0; + PyTypeObject *tp = Py_TYPE(self); tp->tp_clear((PyObject *)self); - connection_close(self); - self->initialized = 0; + if (connection_close(self) < 0) { + return -1; + } } // Create and configure SQLite database object. @@ -334,7 +337,9 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, self->initialized = 1; if (autocommit == AUTOCOMMIT_DISABLED) { - (void)connection_exec_stmt(self, "BEGIN"); + if (connection_exec_stmt(self, "BEGIN") < 0) { + return -1; + } } return 0; @@ -432,48 +437,83 @@ free_callback_contexts(pysqlite_Connection *self) static void remove_callbacks(sqlite3 *db) { - sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0); + /* None of these APIs can fail, as long as they are given a valid + * database pointer. */ + assert(db != NULL); + int rc = sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0); + assert(rc == SQLITE_OK), (void)rc; + sqlite3_progress_handler(db, 0, 0, (void *)0); - (void)sqlite3_set_authorizer(db, NULL, NULL); + + rc = sqlite3_set_authorizer(db, NULL, NULL); + assert(rc == SQLITE_OK), (void)rc; } -static void +static int connection_close(pysqlite_Connection *self) { - if (self->db) { - if (self->autocommit == AUTOCOMMIT_DISABLED && - !sqlite3_get_autocommit(self->db)) - { - /* If close is implicitly called as a result of interpreter - * tear-down, we must not call back into Python. */ - if (_Py_IsInterpreterFinalizing(PyInterpreterState_Get())) { - remove_callbacks(self->db); - } - (void)connection_exec_stmt(self, "ROLLBACK"); + if (self->db == NULL) { + return 0; + } + + int rc = 0; + if (self->autocommit == AUTOCOMMIT_DISABLED && + !sqlite3_get_autocommit(self->db)) + { + if (connection_exec_stmt(self, "ROLLBACK") < 0) { + rc = -1; } + } - free_callback_contexts(self); + sqlite3 *db = self->db; + self->db = NULL; - sqlite3 *db = self->db; - self->db = NULL; + Py_BEGIN_ALLOW_THREADS + /* The v2 close call always returns SQLITE_OK if given a valid database + * pointer (which we do), so we can safely ignore the return value */ + (void)sqlite3_close_v2(db); + Py_END_ALLOW_THREADS - Py_BEGIN_ALLOW_THREADS - int rc = sqlite3_close_v2(db); - assert(rc == SQLITE_OK), (void)rc; - Py_END_ALLOW_THREADS - } + free_callback_contexts(self); + return rc; } static void -connection_dealloc(pysqlite_Connection *self) +connection_finalize(PyObject *self) { - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + pysqlite_Connection *con = (pysqlite_Connection *)self; + PyObject *exc = PyErr_GetRaisedException(); + + /* If close is implicitly called as a result of interpreter + * tear-down, we must not call back into Python. */ + PyInterpreterState *interp = PyInterpreterState_Get(); + int teardown = _Py_IsInterpreterFinalizing(interp); + if (teardown && con->db) { + remove_callbacks(con->db); + } /* Clean up if user has not called .close() explicitly. */ - connection_close(self); + if (connection_close(con) < 0) { + if (teardown) { + PyErr_Clear(); + } + else { + PyErr_WriteUnraisable((PyObject *)self); + } + } + + PyErr_SetRaisedException(exc); +} +static void +connection_dealloc(PyObject *self) +{ + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + return; + } + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -621,7 +661,9 @@ pysqlite_connection_close_impl(pysqlite_Connection *self) pysqlite_close_all_blobs(self); Py_CLEAR(self->statement_cache); - connection_close(self); + if (connection_close(self) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -2555,6 +2597,7 @@ static struct PyMemberDef connection_members[] = }; static PyType_Slot connection_slots[] = { + {Py_tp_finalize, connection_finalize}, {Py_tp_dealloc, connection_dealloc}, {Py_tp_doc, (void *)connection_doc}, {Py_tp_methods, connection_methods}, From 28cab71f954f3a14de9f474ce9c4abbd23c97862 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 18 Aug 2023 22:42:45 +0900 Subject: [PATCH 188/598] gh-104504: Run mypy on cases_generator in CI (and blacken the code) (gh-108090) Co-authored-by: Alex Waygood --- .github/dependabot.yml | 2 +- .github/workflows/mypy.yml | 14 +- Tools/cases_generator/flags.py | 17 +- Tools/cases_generator/formatting.py | 11 +- Tools/cases_generator/generate_cases.py | 97 +++---- Tools/cases_generator/lexer.py | 329 +++++++++++++++--------- Tools/cases_generator/mypy.ini | 14 + Tools/cases_generator/parsing.py | 4 +- Tools/cases_generator/stacking.py | 16 +- Tools/clinic/requirements-dev.txt | 2 - Tools/requirements-dev.txt | 3 + 11 files changed, 314 insertions(+), 195 deletions(-) create mode 100644 Tools/cases_generator/mypy.ini delete mode 100644 Tools/clinic/requirements-dev.txt create mode 100644 Tools/requirements-dev.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f026b0f5f9454a..c8a3165d690364 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,7 +13,7 @@ updates: - "version-update:semver-minor" - "version-update:semver-patch" - package-ecosystem: "pip" - directory: "/Tools/clinic/" + directory: "/Tools/" schedule: interval: "monthly" labels: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1315bb5a966f01..a83a90c69529f2 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - "Tools/clinic/**" + - "Tools/cases_generator/**" - ".github/workflows/mypy.yml" workflow_dispatch: @@ -25,15 +26,18 @@ concurrency: jobs: mypy: - name: Run mypy on Tools/clinic/ + strategy: + matrix: + target: ["Tools/cases_generator", "Tools/clinic"] + name: Run mypy on ${{ matrix.target }} runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: "3.11" cache: pip - cache-dependency-path: Tools/clinic/requirements-dev.txt - - run: pip install -r Tools/clinic/requirements-dev.txt - - run: mypy --config-file Tools/clinic/mypy.ini + cache-dependency-path: Tools/requirements-dev.txt + - run: pip install -r Tools/requirements-dev.txt + - run: mypy --config-file ${{ matrix.target }}/mypy.ini diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 962f003b194dbd..536f093c7d6ede 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -16,12 +16,11 @@ class InstructionFlags: HAS_FREE_FLAG: bool HAS_LOCAL_FLAG: bool - def __post_init__(self): + def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @staticmethod - def fromInstruction(instr: parsing.Node): - + def fromInstruction(instr: parsing.Node) -> "InstructionFlags": has_free = ( variable_used(instr, "PyCell_New") or variable_used(instr, "PyCell_GET") @@ -41,7 +40,7 @@ def fromInstruction(instr: parsing.Node): ) @staticmethod - def newEmpty(): + def newEmpty() -> "InstructionFlags": return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: @@ -49,7 +48,7 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value=None) -> list[str]: + def names(self, value: bool | None = None) -> list[str]: if value is None: return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] @@ -62,7 +61,7 @@ def bitmap(self) -> int: return flags @classmethod - def emit_macros(cls, out: Formatter): + def emit_macros(cls, out: Formatter) -> None: flags = cls.newEmpty() for name, value in flags.bitmask.items(): out.emit(f"#define {name} ({value})") @@ -90,9 +89,9 @@ def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: text = "".join(token.text.split()) # TODO: Handle nested #if if text == "#if": - if ( - i + 1 < len(node.tokens) - and node.tokens[i + 1].text in ("ENABLE_SPECIALIZATION", "TIER_ONE") + if i + 1 < len(node.tokens) and node.tokens[i + 1].text in ( + "ENABLE_SPECIALIZATION", + "TIER_ONE", ): skipping = True elif text in ("#else", "#endif"): diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index 5894751bd9635c..4fd9172d20c274 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -1,6 +1,7 @@ import contextlib import re import typing +from collections.abc import Iterator from parsing import StackEffect, Family @@ -58,13 +59,13 @@ def reset_lineno(self) -> None: self.set_lineno(self.lineno + 1, self.filename) @contextlib.contextmanager - def indent(self): + def indent(self) -> Iterator[None]: self.prefix += " " yield self.prefix = self.prefix[:-4] @contextlib.contextmanager - def block(self, head: str, tail: str = ""): + def block(self, head: str, tail: str = "") -> Iterator[None]: if head: self.emit(head + " {") else: @@ -77,7 +78,7 @@ def stack_adjust( self, input_effects: list[StackEffect], output_effects: list[StackEffect], - ): + ) -> None: shrink, isym = list_effect_size(input_effects) grow, osym = list_effect_size(output_effects) diff = grow - shrink @@ -90,7 +91,7 @@ def stack_adjust( if osym and osym != isym: self.emit(f"STACK_GROW({osym});") - def declare(self, dst: StackEffect, src: StackEffect | None): + def declare(self, dst: StackEffect, src: StackEffect | None) -> None: if dst.name == UNUSED or dst.cond == "0": return typ = f"{dst.type}" if dst.type else "PyObject *" @@ -107,7 +108,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None): sepa = "" if typ.endswith("*") else " " self.emit(f"{typ}{sepa}{dst.name}{init};") - def assign(self, dst: StackEffect, src: StackEffect): + def assign(self, dst: StackEffect, src: StackEffect) -> None: if src.name == UNUSED or dst.name == UNUSED: return cast = self.cast(dst, src) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f31b6658d6599e..de31129ac0548d 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -10,6 +10,7 @@ import posixpath import sys import typing +from collections.abc import Iterator import stacking # Early import to avoid circular import from analysis import Analyzer @@ -23,7 +24,6 @@ MacroInstruction, MacroParts, PseudoInstruction, - StackEffect, OverriddenInstructionPlaceHolder, TIER_ONE, TIER_TWO, @@ -80,12 +80,10 @@ "STORE_FAST", "STORE_FAST_MAYBE_NULL", "COPY", - # Arithmetic "_BINARY_OP_MULTIPLY_INT", "_BINARY_OP_ADD_INT", "_BINARY_OP_SUBTRACT_INT", - } arg_parser = argparse.ArgumentParser( @@ -144,6 +142,7 @@ default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, ) + class Generator(Analyzer): def get_stack_effect_info( self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo @@ -183,7 +182,8 @@ def effect_str(effects: list[StackEffect]) -> str: assert target_instr target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) - if popped is None and pushed is None: + if pushed is None: + assert popped is None popped, pushed = target_popped, target_pushed else: assert popped == target_popped @@ -193,7 +193,7 @@ def effect_str(effects: list[StackEffect]) -> str: return instr, popped, pushed @contextlib.contextmanager - def metadata_item(self, signature, open, close): + def metadata_item(self, signature: str, open: str, close: str) -> Iterator[None]: self.out.emit("") self.out.emit(f"extern {signature};") self.out.emit("#ifdef NEED_OPCODE_METADATA") @@ -216,9 +216,10 @@ def write_stack_effect_functions(self) -> None: def write_function( direction: str, data: list[tuple[AnyInstruction, str]] ) -> None: - with self.metadata_item( - f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", "", "" + f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", + "", + "", ): with self.out.block("switch(opcode)"): for instr, effect in data: @@ -243,23 +244,24 @@ def from_source_files(self) -> str: paths = f"\n{self.out.comment} ".join(filenames) return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" - def write_provenance_header(self): + def write_provenance_header(self) -> None: self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") self.out.write_raw(self.from_source_files()) self.out.write_raw(f"{self.out.comment} Do not edit!\n") - def assign_opcode_ids(self): + def assign_opcode_ids(self) -> None: """Assign IDs to opcodes""" - ops: list[(bool, str)] = [] # (has_arg, name) for each opcode + ops: list[tuple[bool, str]] = [] # (has_arg, name) for each opcode instrumented_ops: list[str] = [] for instr in itertools.chain( [instr for instr in self.instrs.values() if instr.kind != "op"], - self.macro_instrs.values()): - + self.macro_instrs.values(), + ): + assert isinstance(instr, (Instruction, MacroInstruction, PseudoInstruction)) name = instr.name - if name.startswith('INSTRUMENTED_'): + if name.startswith("INSTRUMENTED_"): instrumented_ops.append(name) else: ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) @@ -268,33 +270,32 @@ def assign_opcode_ids(self): # rather than bytecodes.c, so we need to add it explicitly # here (at least until we add something to bytecodes.c to # declare external instructions). - instrumented_ops.append('INSTRUMENTED_LINE') + instrumented_ops.append("INSTRUMENTED_LINE") # assert lists are unique assert len(set(ops)) == len(ops) assert len(set(instrumented_ops)) == len(instrumented_ops) - opname: list[str or None] = [None] * 512 - opmap: dict = {} - markers: dict = {} + opname: list[str | None] = [None] * 512 + opmap: dict[str, int] = {} + markers: dict[str, int] = {} - def map_op(op, name): + def map_op(op: int, name: str) -> None: assert op < len(opname) assert opname[op] is None assert name not in opmap opname[op] = name opmap[name] = op - # 0 is reserved for cache entries. This helps debugging. - map_op(0, 'CACHE') + map_op(0, "CACHE") # 17 is reserved as it is the initial value for the specializing counter. # This helps catch cases where we attempt to execute a cache. - map_op(17, 'RESERVED') + map_op(17, "RESERVED") # 166 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py - map_op(166, 'RESUME') + map_op(166, "RESUME") next_opcode = 1 @@ -306,13 +307,13 @@ def map_op(op, name): assert next_opcode < 255 map_op(next_opcode, name) - if has_arg and 'HAVE_ARGUMENT' not in markers: - markers['HAVE_ARGUMENT'] = next_opcode + if has_arg and "HAVE_ARGUMENT" not in markers: + markers["HAVE_ARGUMENT"] = next_opcode # Instrumented opcodes are at the end of the valid range min_instrumented = 254 - (len(instrumented_ops) - 1) assert next_opcode <= min_instrumented - markers['MIN_INSTRUMENTED_OPCODE'] = min_instrumented + markers["MIN_INSTRUMENTED_OPCODE"] = min_instrumented for i, op in enumerate(instrumented_ops): map_op(min_instrumented + i, op) @@ -320,11 +321,13 @@ def map_op(op, name): for i, op in enumerate(sorted(self.pseudos)): map_op(256 + i, op) - assert 255 not in opmap # 255 is reserved + assert 255 not in opmap.values() # 255 is reserved self.opmap = opmap self.markers = markers - def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): + def write_opcode_ids( + self, opcode_ids_h_filename: str, opcode_targets_filename: str + ) -> None: """Write header file that defined the opcode IDs""" with open(opcode_ids_h_filename, "w") as f: @@ -337,15 +340,15 @@ def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): self.out.emit("#ifndef Py_OPCODE_IDS_H") self.out.emit("#define Py_OPCODE_IDS_H") self.out.emit("#ifdef __cplusplus") - self.out.emit("extern \"C\" {") + self.out.emit('extern "C" {') self.out.emit("#endif") self.out.emit("") self.out.emit("/* Instruction opcodes for compiled code */") - def define(name, opcode): + def define(name: str, opcode: int) -> None: self.out.emit(f"#define {name:<38} {opcode:>3}") - all_pairs = [] + all_pairs: list[tuple[int, int, str]] = [] # the second item in the tuple sorts the markers before the ops all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) @@ -370,7 +373,6 @@ def define(name, opcode): targets[op] = f"TARGET_{name}" f.write(",\n".join([f" &&{s}" for s in targets])) - def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: """Write instruction metadata to output file.""" @@ -469,7 +471,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No "const struct opcode_metadata " "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", "=", - ";" + ";", ): # Write metadata for each instruction for thing in self.everything: @@ -482,7 +484,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Macro(): self.write_metadata_for_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): - self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) + self.write_metadata_for_pseudo( + self.pseudo_instrs[thing.name] + ) case _: typing.assert_never(thing) @@ -490,7 +494,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No "const struct opcode_macro_expansion " "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", "=", - ";" + ";", ): # Write macro expansion for each non-pseudo instruction for thing in self.everything: @@ -529,7 +533,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') with self.metadata_item( - f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", "=", ";" + f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", + "=", + ";", ): for name in self.opmap: self.out.emit(f'[{name}] = "{name}",') @@ -542,11 +548,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No for m in family.members: deoptcodes[m] = name # special case: - deoptcodes['BINARY_OP_INPLACE_ADD_UNICODE'] = 'BINARY_OP' + deoptcodes["BINARY_OP_INPLACE_ADD_UNICODE"] = "BINARY_OP" - with self.metadata_item( - f"const uint8_t _PyOpcode_Deopt[256]", "=", ";" - ): + with self.metadata_item(f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"): for opt, deopt in sorted(deoptcodes.items()): self.out.emit(f"[{opt}] = {deopt},") @@ -604,10 +608,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No if name not in specialized_ops: self.out.emit(f"'{name}': {op},") - for name in ['MIN_INSTRUMENTED_OPCODE', 'HAVE_ARGUMENT']: + for name in ["MIN_INSTRUMENTED_OPCODE", "HAVE_ARGUMENT"]: self.out.emit(f"{name} = {self.markers[name]}") - def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\") @@ -834,7 +837,10 @@ def write_abstract_interpreter_instructions( pass case parsing.InstDef(): instr = AbstractInstruction(self.instrs[thing.name].inst) - if instr.is_viable_uop() and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR: + if ( + instr.is_viable_uop() + and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR + ): self.out.emit("") with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) @@ -878,7 +884,7 @@ def write_instr(self, instr: Instruction) -> None: self.out.emit(f"DISPATCH();") -def main(): +def main() -> None: """Parse command line, parse input, analyze, write output.""" args = arg_parser.parse_args() # Prints message and sys.exit(2) on error if len(args.input) == 0: @@ -899,8 +905,9 @@ def main(): a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) - a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, - args.emit_line_directives) + a.write_abstract_interpreter_instructions( + args.abstract_interpreter_cases, args.emit_line_directives + ) if __name__ == "__main__": diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index fe9c05ede5aa47..a60f6c11a4c460 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -4,132 +4,221 @@ import re from dataclasses import dataclass +from collections.abc import Iterator -def choice(*opts): + +def choice(*opts: str) -> str: return "|".join("(%s)" % opt for opt in opts) + # Regexes # Longer operators must go before shorter ones. -PLUSPLUS = r'\+\+' -MINUSMINUS = r'--' +PLUSPLUS = r"\+\+" +MINUSMINUS = r"--" # -> -ARROW = r'->' -ELLIPSIS = r'\.\.\.' +ARROW = r"->" +ELLIPSIS = r"\.\.\." # Assignment operators -TIMESEQUAL = r'\*=' -DIVEQUAL = r'/=' -MODEQUAL = r'%=' -PLUSEQUAL = r'\+=' -MINUSEQUAL = r'-=' -LSHIFTEQUAL = r'<<=' -RSHIFTEQUAL = r'>>=' -ANDEQUAL = r'&=' -OREQUAL = r'\|=' -XOREQUAL = r'\^=' +TIMESEQUAL = r"\*=" +DIVEQUAL = r"/=" +MODEQUAL = r"%=" +PLUSEQUAL = r"\+=" +MINUSEQUAL = r"-=" +LSHIFTEQUAL = r"<<=" +RSHIFTEQUAL = r">>=" +ANDEQUAL = r"&=" +OREQUAL = r"\|=" +XOREQUAL = r"\^=" # Operators -PLUS = r'\+' -MINUS = r'-' -TIMES = r'\*' -DIVIDE = r'/' -MOD = r'%' -NOT = r'~' -XOR = r'\^' -LOR = r'\|\|' -LAND = r'&&' -LSHIFT = r'<<' -RSHIFT = r'>>' -LE = r'<=' -GE = r'>=' -EQ = r'==' -NE = r'!=' -LT = r'<' -GT = r'>' -LNOT = r'!' -OR = r'\|' -AND = r'&' -EQUALS = r'=' +PLUS = r"\+" +MINUS = r"-" +TIMES = r"\*" +DIVIDE = r"/" +MOD = r"%" +NOT = r"~" +XOR = r"\^" +LOR = r"\|\|" +LAND = r"&&" +LSHIFT = r"<<" +RSHIFT = r">>" +LE = r"<=" +GE = r">=" +EQ = r"==" +NE = r"!=" +LT = r"<" +GT = r">" +LNOT = r"!" +OR = r"\|" +AND = r"&" +EQUALS = r"=" # ? -CONDOP = r'\?' +CONDOP = r"\?" # Delimiters -LPAREN = r'\(' -RPAREN = r'\)' -LBRACKET = r'\[' -RBRACKET = r'\]' -LBRACE = r'\{' -RBRACE = r'\}' -COMMA = r',' -PERIOD = r'\.' -SEMI = r';' -COLON = r':' -BACKSLASH = r'\\' - -operators = { op: pattern for op, pattern in globals().items() if op == op.upper() } +LPAREN = r"\(" +RPAREN = r"\)" +LBRACKET = r"\[" +RBRACKET = r"\]" +LBRACE = r"\{" +RBRACE = r"\}" +COMMA = r"," +PERIOD = r"\." +SEMI = r";" +COLON = r":" +BACKSLASH = r"\\" + +operators = {op: pattern for op, pattern in globals().items() if op == op.upper()} for op in operators: globals()[op] = op -opmap = { pattern.replace("\\", "") or '\\' : op for op, pattern in operators.items() } +opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r'# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)' -MACRO = 'MACRO' +macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" +MACRO = "MACRO" -id_re = r'[a-zA-Z_][0-9a-zA-Z_]*' -IDENTIFIER = 'IDENTIFIER' +id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" +IDENTIFIER = "IDENTIFIER" -suffix = r'([uU]?[lL]?[lL]?)' -octal = r'0[0-7]+' + suffix -hex = r'0[xX][0-9a-fA-F]+' -decimal_digits = r'(0|[1-9][0-9]*)' +suffix = r"([uU]?[lL]?[lL]?)" +octal = r"0[0-7]+" + suffix +hex = r"0[xX][0-9a-fA-F]+" +decimal_digits = r"(0|[1-9][0-9]*)" decimal = decimal_digits + suffix exponent = r"""([eE][-+]?[0-9]+)""" fraction = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" -float = '(((('+fraction+')'+exponent+'?)|([0-9]+'+exponent+'))[FfLl]?)' +float = "((((" + fraction + ")" + exponent + "?)|([0-9]+" + exponent + "))[FfLl]?)" number_re = choice(octal, hex, float, decimal) -NUMBER = 'NUMBER' +NUMBER = "NUMBER" simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" decimal_escape = r"""(\d+)""" hex_escape = r"""(x[0-9a-fA-F]+)""" -escape_sequence = r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))' -string_char = r"""([^"\\\n]|"""+escape_sequence+')' -str_re = '"'+string_char+'*"' -STRING = 'STRING' -char = r'\'.\'' # TODO: escape sequence -CHARACTER = 'CHARACTER' +escape_sequence = ( + r"""(\\(""" + simple_escape + "|" + decimal_escape + "|" + hex_escape + "))" +) +string_char = r"""([^"\\\n]|""" + escape_sequence + ")" +str_re = '"' + string_char + '*"' +STRING = "STRING" +char = r"\'.\'" # TODO: escape sequence +CHARACTER = "CHARACTER" -comment_re = r'//.*|/\*([^*]|\*[^/])*\*/' -COMMENT = 'COMMENT' +comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +COMMENT = "COMMENT" newline = r"\n" -invalid = r"\S" # A single non-space character that's not caught by any of the other patterns -matcher = re.compile(choice(id_re, number_re, str_re, char, newline, macro, comment_re, *operators.values(), invalid)) -letter = re.compile(r'[a-zA-Z_]') - -kwds = ( - 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', - 'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN', - 'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', 'OVERRIDE', - 'REGISTER', 'OFFSETOF', - 'RESTRICT', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT', - 'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID', - 'VOLATILE', 'WHILE' +invalid = ( + r"\S" # A single non-space character that's not caught by any of the other patterns ) -for name in kwds: - globals()[name] = name -keywords = { name.lower() : name for name in kwds } +matcher = re.compile( + choice( + id_re, + number_re, + str_re, + char, + newline, + macro, + comment_re, + *operators.values(), + invalid, + ) +) +letter = re.compile(r"[a-zA-Z_]") + + +kwds = [] +AUTO = "AUTO" +kwds.append(AUTO) +BREAK = "BREAK" +kwds.append(BREAK) +CASE = "CASE" +kwds.append(CASE) +CHAR = "CHAR" +kwds.append(CHAR) +CONST = "CONST" +kwds.append(CONST) +CONTINUE = "CONTINUE" +kwds.append(CONTINUE) +DEFAULT = "DEFAULT" +kwds.append(DEFAULT) +DO = "DO" +kwds.append(DO) +DOUBLE = "DOUBLE" +kwds.append(DOUBLE) +ELSE = "ELSE" +kwds.append(ELSE) +ENUM = "ENUM" +kwds.append(ENUM) +EXTERN = "EXTERN" +kwds.append(EXTERN) +FLOAT = "FLOAT" +kwds.append(FLOAT) +FOR = "FOR" +kwds.append(FOR) +GOTO = "GOTO" +kwds.append(GOTO) +IF = "IF" +kwds.append(IF) +INLINE = "INLINE" +kwds.append(INLINE) +INT = "INT" +kwds.append(INT) +LONG = "LONG" +kwds.append(LONG) +OVERRIDE = "OVERRIDE" +kwds.append(OVERRIDE) +REGISTER = "REGISTER" +kwds.append(REGISTER) +OFFSETOF = "OFFSETOF" +kwds.append(OFFSETOF) +RESTRICT = "RESTRICT" +kwds.append(RESTRICT) +RETURN = "RETURN" +kwds.append(RETURN) +SHORT = "SHORT" +kwds.append(SHORT) +SIGNED = "SIGNED" +kwds.append(SIGNED) +SIZEOF = "SIZEOF" +kwds.append(SIZEOF) +STATIC = "STATIC" +kwds.append(STATIC) +STRUCT = "STRUCT" +kwds.append(STRUCT) +SWITCH = "SWITCH" +kwds.append(SWITCH) +TYPEDEF = "TYPEDEF" +kwds.append(TYPEDEF) +UNION = "UNION" +kwds.append(UNION) +UNSIGNED = "UNSIGNED" +kwds.append(UNSIGNED) +VOID = "VOID" +kwds.append(VOID) +VOLATILE = "VOLATILE" +kwds.append(VOLATILE) +WHILE = "WHILE" +kwds.append(WHILE) +keywords = {name.lower(): name for name in kwds} + +__all__ = [] +__all__.extend(kwds) def make_syntax_error( - message: str, filename: str, line: int, column: int, line_text: str, + message: str, + filename: str | None, + line: int, + column: int, + line_text: str, ) -> SyntaxError: return SyntaxError(message, (filename, line, column, line_text)) @@ -142,30 +231,30 @@ class Token: end: tuple[int, int] @property - def line(self): + def line(self) -> int: return self.begin[0] @property - def column(self): + def column(self) -> int: return self.begin[1] @property - def end_line(self): + def end_line(self) -> int: return self.end[0] @property - def end_column(self): + def end_column(self) -> int: return self.end[1] @property - def width(self): + def width(self) -> int: return self.end[1] - self.begin[1] - def replaceText(self, txt): + def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) return Token(self.kind, txt, self.begin, self.end) - def __repr__(self): + def __repr__(self) -> str: b0, b1 = self.begin e0, e1 = self.end if b0 == e0: @@ -174,7 +263,7 @@ def __repr__(self): return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src, line=1, filename=None): +def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() @@ -183,73 +272,75 @@ def tokenize(src, line=1, filename=None): kind = keywords[text] elif letter.match(text): kind = IDENTIFIER - elif text == '...': + elif text == "...": kind = ELLIPSIS - elif text == '.': + elif text == ".": kind = PERIOD - elif text[0] in '0123456789.': + elif text[0] in "0123456789.": kind = NUMBER elif text[0] == '"': kind = STRING elif text in opmap: kind = opmap[text] - elif text == '\n': + elif text == "\n": linestart = start line += 1 - kind = '\n' + kind = "\n" elif text[0] == "'": kind = CHARACTER - elif text[0] == '#': + elif text[0] == "#": kind = MACRO - elif text[0] == '/' and text[1] in '/*': + elif text[0] == "/" and text[1] in "/*": kind = COMMENT else: lineend = src.find("\n", start) if lineend == -1: lineend = len(src) - raise make_syntax_error(f"Bad token: {text}", - filename, line, start-linestart+1, src[linestart:lineend]) + raise make_syntax_error( + f"Bad token: {text}", + filename, + line, + start - linestart + 1, + src[linestart:lineend], + ) if kind == COMMENT: - begin = line, start-linestart - newlines = text.count('\n') + begin = line, start - linestart + newlines = text.count("\n") if newlines: - linestart = start + text.rfind('\n') + linestart = start + text.rfind("\n") line += newlines else: - begin = line, start-linestart + begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start-linestart+len(text))) - - -__all__ = [] -__all__.extend([kind for kind in globals() if kind.upper() == kind]) + yield Token(kind, text, begin, (line, start - linestart + len(text))) def to_text(tkns: list[Token], dedent: int = 0) -> str: res: list[str] = [] - line, col = -1, 1+dedent + line, col = -1, 1 + dedent for tkn in tkns: if line == -1: line, _ = tkn.begin l, c = tkn.begin - #assert(l >= line), (line, txt, start, end) + # assert(l >= line), (line, txt, start, end) while l > line: line += 1 - res.append('\n') - col = 1+dedent - res.append(' '*(c-col)) + res.append("\n") + col = 1 + dedent + res.append(" " * (c - col)) text = tkn.text - if dedent != 0 and tkn.kind == 'COMMENT' and '\n' in text: + if dedent != 0 and tkn.kind == "COMMENT" and "\n" in text: if dedent < 0: - text = text.replace('\n', '\n' + ' '*-dedent) + text = text.replace("\n", "\n" + " " * -dedent) # TODO: dedent > 0 res.append(text) line, col = tkn.end - return ''.join(res) + return "".join(res) if __name__ == "__main__": import sys + filename = sys.argv[1] if filename == "-c": src = sys.argv[2] diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini new file mode 100644 index 00000000000000..7480841bf07edc --- /dev/null +++ b/Tools/cases_generator/mypy.ini @@ -0,0 +1,14 @@ +[mypy] +files = Tools/cases_generator/ +pretty = True + +python_version = 3.10 + +# Be strict: +strict = True +strict_concatenate = True +enable_error_code = ignore-without-code,redundant-expr,truthy-bool + +# Don't enable this one yet; +# it has a lot of false positives on `cases_generator` +warn_unreachable = False diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index cdd20d7a0b3f59..25de3a5adc2cf9 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -32,7 +32,7 @@ class Context(NamedTuple): end: int owner: PLexer - def __repr__(self): + def __repr__(self) -> str: return f"<{self.owner.filename}: {self.begin}-{self.end}>" @@ -75,7 +75,7 @@ class StackEffect(Node): size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond - def __repr__(self): + def __repr__(self) -> str: items = [self.name, self.type, self.cond, self.size] while items and items[-1] == "": del items[-1] diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 632298a567dd40..1e117f11825938 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -413,22 +413,22 @@ def write_components( return next_instr_is_set -def write_single_instr_for_abstract_interp( - instr: Instruction, out: Formatter -): +def write_single_instr_for_abstract_interp(instr: Instruction, out: Formatter) -> None: try: _write_components_for_abstract_interp( [Component(instr, instr.active_caches)], out, ) except AssertionError as err: - raise AssertionError(f"Error writing abstract instruction {instr.name}") from err + raise AssertionError( + f"Error writing abstract instruction {instr.name}" + ) from err def _write_components_for_abstract_interp( parts: list[Component], out: Formatter, -): +) -> None: managers = get_managers(parts) for mgr in managers: if mgr is managers[-1]: @@ -438,5 +438,7 @@ def _write_components_for_abstract_interp( # NULL out the output stack effects for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.emit(f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" - f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);") + out.emit( + f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" + f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);" + ) diff --git a/Tools/clinic/requirements-dev.txt b/Tools/clinic/requirements-dev.txt deleted file mode 100644 index e9529f3527e95e..00000000000000 --- a/Tools/clinic/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Requirements file for external linters and checks we run on Tools/clinic/ in CI -mypy==1.4.1 diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt new file mode 100644 index 00000000000000..111773f4f47378 --- /dev/null +++ b/Tools/requirements-dev.txt @@ -0,0 +1,3 @@ +# Requirements file for external linters and checks we run on +# Tools/clinic and Tools/cases_generator/ in CI +mypy==1.5.1 From 9bb576cb07b42f34fd882b8502642b024f771c62 Mon Sep 17 00:00:00 2001 From: Tyler Smart <40041862+tjsmart@users.noreply.github.com> Date: Fri, 18 Aug 2023 08:44:38 -0700 Subject: [PATCH 189/598] gh-107995: Fix doctest collection of functools.cached_property objects (#107996) Co-authored-by: Alex Waygood --- Lib/functools.py | 1 + Lib/test/test_doctest.py | 12 ++++++++++++ Lib/test/test_functools.py | 3 +++ Misc/ACKS | 1 + .../2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst | 5 +++++ 5 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst diff --git a/Lib/functools.py b/Lib/functools.py index be44ccdae6b692..a2fc28779dbddc 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -984,6 +984,7 @@ def __init__(self, func): self.func = func self.attrname = None self.__doc__ = func.__doc__ + self.__module__ = func.__module__ def __set_name__(self, owner, name): if self.attrname is None: diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index bea52c6de7ec6d..c364e81a6b1495 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -111,6 +111,14 @@ def a_classmethod_property(cls): """ return cls.a_class_attribute + @functools.cached_property + def a_cached_property(self): + """ + >>> print(SampleClass(29).get()) + 29 + """ + return "hello" + class NestedClass: """ >>> x = SampleClass.NestedClass(5) @@ -515,6 +523,7 @@ def basics(): r""" 3 SampleClass.NestedClass 1 SampleClass.NestedClass.__init__ 1 SampleClass.__init__ + 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod 1 SampleClass.a_classmethod_property 1 SampleClass.a_property @@ -571,6 +580,7 @@ def basics(): r""" 3 some_module.SampleClass.NestedClass 1 some_module.SampleClass.NestedClass.__init__ 1 some_module.SampleClass.__init__ + 1 some_module.SampleClass.a_cached_property 2 some_module.SampleClass.a_classmethod 1 some_module.SampleClass.a_classmethod_property 1 some_module.SampleClass.a_property @@ -613,6 +623,7 @@ def basics(): r""" 3 SampleClass.NestedClass 1 SampleClass.NestedClass.__init__ 1 SampleClass.__init__ + 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod 1 SampleClass.a_classmethod_property 1 SampleClass.a_property @@ -634,6 +645,7 @@ def basics(): r""" 0 SampleClass.NestedClass.get 0 SampleClass.NestedClass.square 1 SampleClass.__init__ + 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod 1 SampleClass.a_classmethod_property 1 SampleClass.a_property diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 50770f066a5e16..5ba7f51c91f3b5 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -3105,6 +3105,9 @@ def test_access_from_class(self): def test_doc(self): self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + def test_module(self): + self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) + def test_subclass_with___set__(self): """Caching still works for a subclass defining __set__.""" class readonly_cached_property(py_functools.cached_property): diff --git a/Misc/ACKS b/Misc/ACKS index 8b8c5ad8434bd7..475c6ec3dab73b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1706,6 +1706,7 @@ Roman Skurikhin Ville Skyttä Michael Sloan Nick Sloan +Tyler Smart Radek Smejkal Václav Šmilauer Casper W. Smet diff --git a/Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst b/Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst new file mode 100644 index 00000000000000..7247f6b3abc9bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst @@ -0,0 +1,5 @@ +The ``__module__`` attribute on instances of :class:`functools.cached_property` +is now set to the name of the module in which the cached_property is defined, +rather than "functools". This means that doctests in ``cached_property`` +docstrings are now properly collected by the :mod:`doctest` module. Patch by +Tyler Smart. From 6db39b1460727e8820b45e5c083e5839af0352aa Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 18 Aug 2023 12:13:58 -0500 Subject: [PATCH 190/598] Minor code clean-up for the factor() recipe (GH-108114) --- Doc/library/itertools.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 730736bbb59ed9..d0bb4696970198 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1048,9 +1048,7 @@ The following recipes have a more mathematical flavor: # factor(1_000_000_000_000_007) --> 47 59 360620266859 # factor(1_000_000_000_000_403) --> 1000000000000403 for prime in sieve(math.isqrt(n) + 1): - while True: - if n % prime: - break + while not n % prime: yield prime n //= prime if n == 1: From dd4442c8f597af1ec3eaf20f7ad89c4ac7e2dbc9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 18 Aug 2023 19:53:51 +0200 Subject: [PATCH 191/598] gh-107801: Improve the accuracy of os.lseek docs (#107935) - name the last parameter *whence*, like it is for seek() methods on file objects - add param docstrings - structure the valid *whence* params Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/os.rst | 17 +++++++++++------ Modules/clinic/posixmodule.c.h | 17 +++++++++++++---- Modules/posixmodule.c | 13 +++++++++---- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4668984b10c690..7d63ac15027d51 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1163,17 +1163,22 @@ as internal buffering of data. .. versionadded:: 3.11 -.. function:: lseek(fd, pos, how, /) +.. function:: lseek(fd, pos, whence, /) Set the current position of file descriptor *fd* to position *pos*, modified - by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the - beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of - the file. Return the new cursor position in bytes, starting from the beginning. + by *whence*, and return the new position in bytes relative to + the start of the file. + Valid values for *whence* are: + + * :const:`SEEK_SET` or ``0`` -- set *pos* relative to the beginning of the file + * :const:`SEEK_CUR` or ``1`` -- set *pos* relative to the current file position + * :const:`SEEK_END` or ``2`` -- set *pos* relative to the end of the file + * :const:`SEEK_HOLE` -- set *pos* to the next data location, relative to *pos* + * :const:`SEEK_DATA` -- set *pos* to the next data hole, relative to *pos* .. versionchanged:: 3.3 - Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + Add support for :const:`!SEEK_HOLE` and :const:`!SEEK_DATA`. .. data:: SEEK_SET diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 81e3162e679d16..9b3fa8ded26766 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6496,13 +6496,22 @@ os_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_LOCKF) */ PyDoc_STRVAR(os_lseek__doc__, -"lseek($module, fd, position, how, /)\n" +"lseek($module, fd, position, whence, /)\n" "--\n" "\n" "Set the position of a file descriptor. Return the new position.\n" "\n" -"Return the new cursor position in number of bytes\n" -"relative to the beginning of the file."); +" fd\n" +" An open file descriptor, as returned by os.open().\n" +" position\n" +" Position, interpreted relative to \'whence\'.\n" +" whence\n" +" The relative position to seek from. Valid values are:\n" +" - SEEK_SET: seek from the start of the file.\n" +" - SEEK_CUR: seek from the current file position.\n" +" - SEEK_END: seek from the end of the file.\n" +"\n" +"The return value is the number of bytes relative to the beginning of the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -11981,4 +11990,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a85a386b212b0631 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9a5f78bb65470528 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c391aab6fdce1d..8026080912a7d8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10424,19 +10424,24 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) os.lseek -> Py_off_t fd: int + An open file descriptor, as returned by os.open(). position: Py_off_t - how: int + Position, interpreted relative to 'whence'. + whence as how: int + The relative position to seek from. Valid values are: + - SEEK_SET: seek from the start of the file. + - SEEK_CUR: seek from the current file position. + - SEEK_END: seek from the end of the file. / Set the position of a file descriptor. Return the new position. -Return the new cursor position in number of bytes -relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=902654ad3f96a6d3]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=f096e754c5367504]*/ { Py_off_t result; From ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 18 Aug 2023 15:48:20 -0400 Subject: [PATCH 192/598] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-107896) --- .github/workflows/build.yml | 8 ++++---- .../2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst | 2 ++ Tools/ssl/multissltests.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06551b13219c2a..d37eb4447ad11a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -244,7 +244,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 @@ -313,7 +313,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -365,7 +365,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 @@ -474,7 +474,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 00000000000000..c43ee680e8158e --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index f9809c9b54665b..fc261c770d6944 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,9 +46,9 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", - "3.1.1", + "1.1.1v", + "3.0.10", + "3.1.2", ] LIBRESSL_OLD_VERSIONS = [ From dc7b630b2359663bb7b8212d9f2f720c978d3daa Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 18 Aug 2023 17:38:24 -0400 Subject: [PATCH 193/598] gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0fc25ec74565df..c108ee21a7a2a3 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.9", - url="https://www.openssl.org/source/openssl-3.0.9.tar.gz", - checksum='eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90', + name="OpenSSL 3.0.10", + url="https://www.openssl.org/source/openssl-3.0.10.tar.gz", + checksum='1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst new file mode 100644 index 00000000000000..c238c4760239e1 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.10. From eb953d6e4484339067837020f77eecac61f8d4f8 Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Fri, 18 Aug 2023 19:43:28 -0500 Subject: [PATCH 194/598] gh-101100: Only show GitHub check annotations on changed doc paragraphs (#108065) * Only show GitHub check annotations on changed doc paragraphs * Improve check-warnings script arg parsing following Hugo's suggestions * Factor filtering warnings by modified diffs into helper function * Build docs on unmerged branch so warning lines match & avoid deep clone --------- Co-authored-by: Hugo van Kemenade Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .github/workflows/reusable-docs.yml | 33 +++-- Doc/tools/check-warnings.py | 206 +++++++++++++++++++++++++--- 2 files changed, 208 insertions(+), 31 deletions(-) diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 39e5ad62924ad3..6150b1a7d416a3 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -16,8 +16,30 @@ jobs: name: 'Docs' runs-on: ubuntu-latest timeout-minutes: 60 + env: + branch_base: 'origin/${{ github.event.pull_request.base.ref }}' + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - - uses: actions/checkout@v3 + - name: 'Check out latest PR branch commit' + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: 'Fetch commits to get branch diff' + run: | + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: master or main): + git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules - name: 'Set up Python' uses: actions/setup-python@v4 with: @@ -28,13 +50,6 @@ jobs: run: make -C Doc/ venv # To annotate PRs with Sphinx nitpicks (missing references) - - name: 'Get list of changed files' - if: github.event_name == 'pull_request' - id: changed_files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: "Doc/**" - format: csv # works for paths with spaces - name: 'Build HTML documentation' continue-on-error: true run: | @@ -45,7 +60,7 @@ jobs: if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --check-and-annotate '${{ steps.changed_files.outputs.added_modified }}' \ + --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ --fail-if-regression \ --fail-if-improved diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index c17d0f51cd1272..809a8d63087e12 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -2,12 +2,16 @@ """ Check the output of running Sphinx in nit-picky mode (missing references). """ +from __future__ import annotations + import argparse -import csv +import itertools import os import re +import subprocess import sys from pathlib import Path +from typing import TextIO # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. @@ -24,28 +28,178 @@ "venv", } -PATTERN = re.compile(r"(?P[^:]+):(?P\d+): WARNING: (?P.+)") +# Regex pattern to match the parts of a Sphinx warning +WARNING_PATTERN = re.compile( + r"(?P([A-Za-z]:[\\/])?[^:]+):(?P\d+): WARNING: (?P.+)" +) + +# Regex pattern to match the line numbers in a Git unified diff +DIFF_PATTERN = re.compile( + r"^@@ -(?P\d+)(?:,(?P\d+))? \+(?P\d+)(?:,(?P\d+))? @@", + flags=re.MULTILINE, +) + + +def get_diff_files(ref_a: str, ref_b: str, filter_mode: str = "") -> set[Path]: + """List the files changed between two Git refs, filtered by change type.""" + added_files_result = subprocess.run( + [ + "git", + "diff", + f"--diff-filter={filter_mode}", + "--name-only", + f"{ref_a}...{ref_b}", + "--", + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + added_files = added_files_result.stdout.strip().split("\n") + return {Path(file.strip()) for file in added_files if file.strip()} + + +def get_diff_lines(ref_a: str, ref_b: str, file: Path) -> list[int]: + """List the lines changed between two Git refs for a specific file.""" + diff_output = subprocess.run( + [ + "git", + "diff", + "--unified=0", + f"{ref_a}...{ref_b}", + "--", + str(file), + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + # Scrape line offsets + lengths from diff and convert to line numbers + line_matches = DIFF_PATTERN.finditer(diff_output.stdout) + # Removed and added line counts are 1 if not printed + line_match_values = [ + line_match.groupdict(default=1) for line_match in line_matches + ] + line_ints = [ + (int(match_value["lineb"]), int(match_value["added"])) + for match_value in line_match_values + ] + line_ranges = [ + range(line_b, line_b + added) for line_b, added in line_ints + ] + line_numbers = list(itertools.chain(*line_ranges)) + + return line_numbers + + +def get_para_line_numbers(file_obj: TextIO) -> list[list[int]]: + """Get the line numbers of text in a file object, grouped by paragraph.""" + paragraphs = [] + prev_line = None + for lineno, line in enumerate(file_obj): + lineno = lineno + 1 + if prev_line is None or (line.strip() and not prev_line.strip()): + paragraph = [lineno - 1] + paragraphs.append(paragraph) + paragraph.append(lineno) + prev_line = line + return paragraphs + + +def filter_and_parse_warnings( + warnings: list[str], files: set[Path] +) -> list[re.Match[str]]: + """Get the warnings matching passed files and parse them with regex.""" + filtered_warnings = [ + warning + for warning in warnings + if any(str(file) in warning for file in files) + ] + warning_matches = [ + WARNING_PATTERN.fullmatch(warning.strip()) + for warning in filtered_warnings + ] + non_null_matches = [warning for warning in warning_matches if warning] + return non_null_matches + + +def filter_warnings_by_diff( + warnings: list[re.Match[str]], ref_a: str, ref_b: str, file: Path +) -> list[re.Match[str]]: + """Filter the passed per-file warnings to just those on changed lines.""" + diff_lines = get_diff_lines(ref_a, ref_b, file) + with file.open(encoding="UTF-8") as file_obj: + paragraphs = get_para_line_numbers(file_obj) + touched_paras = [ + para_lines + for para_lines in paragraphs + if set(diff_lines) & set(para_lines) + ] + touched_para_lines = set(itertools.chain(*touched_paras)) + warnings_infile = [ + warning for warning in warnings if str(file) in warning["file"] + ] + warnings_touched = [ + warning + for warning in warnings_infile + if int(warning["line"]) in touched_para_lines + ] + return warnings_touched + +def process_touched_warnings( + warnings: list[str], ref_a: str, ref_b: str +) -> list[re.Match[str]]: + """Filter a list of Sphinx warnings to those affecting touched lines.""" + added_files, modified_files = tuple( + get_diff_files(ref_a, ref_b, filter_mode=mode) for mode in ("A", "M") + ) + + warnings_added = filter_and_parse_warnings(warnings, added_files) + warnings_modified = filter_and_parse_warnings(warnings, modified_files) + + modified_files_warned = { + file + for file in modified_files + if any(str(file) in warning["file"] for warning in warnings_modified) + } -def check_and_annotate(warnings: list[str], files_to_check: str) -> None: + warnings_modified_touched = [ + filter_warnings_by_diff(warnings_modified, ref_a, ref_b, file) + for file in modified_files_warned + ] + warnings_touched = warnings_added + list( + itertools.chain(*warnings_modified_touched) + ) + + return warnings_touched + + +def annotate_diff( + warnings: list[str], ref_a: str = "main", ref_b: str = "HEAD" +) -> None: """ - Convert Sphinx warning messages to GitHub Actions. + Convert Sphinx warning messages to GitHub Actions for changed paragraphs. Converts lines like: .../Doc/library/cgi.rst:98: WARNING: reference target not found to: ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found - Non-matching lines are echoed unchanged. - - see: + See: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message """ - files_to_check = next(csv.reader([files_to_check])) - for warning in warnings: - if any(filename in warning for filename in files_to_check): - if match := PATTERN.fullmatch(warning): - print("::warning file={file},line={line}::{msg}".format_map(match)) + warnings_touched = process_touched_warnings(warnings, ref_a, ref_b) + print("Emitting doc warnings matching modified lines:") + for warning in warnings_touched: + print("::warning file={file},line={line}::{msg}".format_map(warning)) + print(warning[0]) + if not warnings_touched: + print("None") def fail_if_regression( @@ -68,7 +222,7 @@ def fail_if_regression( print(filename) for warning in warnings: if filename in warning: - if match := PATTERN.fullmatch(warning): + if match := WARNING_PATTERN.fullmatch(warning): print(" {line}: {msg}".format_map(match)) return -1 return 0 @@ -91,12 +245,14 @@ def fail_if_improved( return 0 -def main() -> int: +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( - "--check-and-annotate", - help="Comma-separated list of files to check, " - "and annotate those with warnings on GitHub Actions", + "--annotate-diff", + nargs="*", + metavar=("BASE_REF", "HEAD_REF"), + help="Add GitHub Actions annotations on the diff for warnings on " + "lines changed between the given refs (main and HEAD, by default)", ) parser.add_argument( "--fail-if-regression", @@ -108,13 +264,19 @@ def main() -> int: action="store_true", help="Fail if new files with no nits are found", ) - args = parser.parse_args() + + args = parser.parse_args(argv) + if args.annotate_diff is not None and len(args.annotate_diff) > 2: + parser.error( + "--annotate-diff takes between 0 and 2 ref args, not " + f"{len(args.annotate_diff)} {tuple(args.annotate_diff)}" + ) exit_code = 0 wrong_directory_msg = "Must run this script from the repo root" assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg - with Path("Doc/sphinx-warnings.txt").open() as f: + with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: warnings = f.read().splitlines() cwd = str(Path.cwd()) + os.path.sep @@ -124,15 +286,15 @@ def main() -> int: if "Doc/" in warning } - with Path("Doc/tools/.nitignore").open() as clean_files: + with Path("Doc/tools/.nitignore").open(encoding="UTF-8") as clean_files: files_with_expected_nits = { filename.strip() for filename in clean_files if filename.strip() and not filename.startswith("#") } - if args.check_and_annotate: - check_and_annotate(warnings, args.check_and_annotate) + if args.annotate_diff is not None: + annotate_diff(warnings, *args.annotate_diff) if args.fail_if_regression: exit_code += fail_if_regression( From 2f311437cd51afaa68fd671bb99ff515cf7b029a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 19 Aug 2023 10:13:35 +0300 Subject: [PATCH 195/598] gh-107704: Argument Clinic: add support for deprecating keyword use of parameters (GH-107984) It is now possible to deprecate passing keyword arguments for keyword-or-positional parameters with Argument Clinic, using the new '/ [from X.Y]' syntax. (To be read as "positional-only from Python version X.Y") Co-authored-by: Erlend E. Aasland Co-authored-by: Alex Waygood --- Doc/howto/clinic.rst | 65 +- Lib/test/test_clinic.py | 373 ++- ...-08-15-19-50-49.gh-issue-107704.Uu84vd.rst | 4 + Modules/_sqlite/clinic/connection.c.h | 37 +- Modules/_testclinic.c | 286 ++- Modules/clinic/_testclinic_depr.c.h | 2095 +++++++++++++++++ Modules/clinic/_testclinic_depr_star.c.h | 1140 --------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 + Tools/clinic/clinic.py | 332 ++- 9 files changed, 2992 insertions(+), 1344 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst create mode 100644 Modules/clinic/_testclinic_depr.c.h delete mode 100644 Modules/clinic/_testclinic_depr_star.c.h diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 463a938fafa8dc..667859e20a5d87 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1941,54 +1941,70 @@ The generated docstring ends up looking like this: .. _clinic-howto-deprecate-positional: +.. _clinic-howto-deprecate-keyword: -How to deprecate passing parameters positionally ------------------------------------------------- +How to deprecate passing parameters positionally or by keyword +-------------------------------------------------------------- Argument Clinic provides syntax that makes it possible to generate code that -deprecates passing :term:`arguments ` positionally. +deprecates passing :term:`arguments ` for positional-or-keyword +:term:`parameters ` positionally or by keyword. For example, say we've got a module-level function :py:func:`!foo.myfunc` -that has three :term:`parameters `: -positional-or-keyword parameters *a* and *b*, and a keyword-only parameter *c*:: +that has five parameters: a positional-only parameter *a*, three +positional-or-keyword parameters *b*, *c* and *d*, and a keyword-only +parameter *e*:: /*[clinic input] module foo myfunc a: int + / b: int - * c: int + d: int + * + e: int [clinic start generated output]*/ -We now want to make the *b* parameter keyword-only; -however, we'll have to wait two releases before making this change, +We now want to make the *b* parameter positional-only and the *d* parameter +keyword-only; +however, we'll have to wait two releases before making these changes, as mandated by Python's backwards-compatibility policy (see :pep:`387`). For this example, imagine we're in the development phase for Python 3.12: that means we'll be allowed to introduce deprecation warnings in Python 3.12 -whenever the *b* parameter is passed positionally, -and we'll be allowed to make it keyword-only in Python 3.14 at the earliest. +whenever an argument for the *b* parameter is passed by keyword or an argument +for the *d* parameter is passed positionally, and we'll be allowed to make +them positional-only and keyword-only respectively in Python 3.14 at +the earliest. We can use Argument Clinic to emit the desired deprecation warnings -using the ``* [from ...]`` syntax, -by adding the line ``* [from 3.14]`` right above the *b* parameter:: +using the ``[from ...]`` syntax, by adding the line ``/ [from 3.14]`` right +below the *b* parameter and adding the line ``* [from 3.14]`` right above +the *d* parameter:: /*[clinic input] module foo myfunc a: int - * [from 3.14] + / b: int - * + / [from 3.14] c: int + * [from 3.14] + d: int + * + e: int [clinic start generated output]*/ Next, regenerate Argument Clinic code (``make clinic``), and add unit tests for the new behaviour. The generated code will now emit a :exc:`DeprecationWarning` -when an :term:`argument` for the :term:`parameter` *b* is passed positionally. +when an :term:`argument` for the :term:`parameter` *d* is passed positionally +(e.g ``myfunc(1, 2, 3, 4, e=5)``) or an argument for the parameter *b* is +passed by keyword (e.g ``myfunc(1, b=2, c=3, d=4, e=5)``). C preprocessor directives are also generated for emitting -compiler warnings if the ``* [from ...]`` line has not been removed +compiler warnings if the ``[from ...]`` lines have not been removed from the Argument Clinic input when the deprecation period is over, which means when the alpha phase of the specified Python version kicks in. @@ -2001,21 +2017,26 @@ Luckily for us, compiler warnings are now generated: .. code-block:: none In file included from Modules/foomodule.c:139: - Modules/clinic/foomodule.c.h:139:8: warning: In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings] - # warning "In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings]" + Modules/clinic/foomodule.c.h:139:8: warning: In 'foomodule.c', update the clinic input of 'mymod.myfunc'. [-W#warnings] + # warning "In 'foomodule.c', update the clinic input of 'mymod.myfunc'. [-W#warnings]" ^ -We now close the deprecation phase by making *b* keyword-only; -replace the ``* [from ...]`` line above *b* -with the ``*`` from the line above *c*:: +We now close the deprecation phase by making *a* positional-only and *c* +keyword-only; +replace the ``/ [from ...]`` line below *b* with the ``/`` from the line +below *a* and the ``* [from ...]`` line above *d* with the ``*`` from +the line above *e*:: /*[clinic input] module foo myfunc a: int - * b: int + / c: int + * + d: int + e: int [clinic start generated output]*/ Finally, run ``make clinic`` to regenerate the Argument Clinic code, diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 38eabd2f7f21dc..934c1e3ffccca8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1611,7 +1611,7 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - err = "Function 'foo.bar' specifies '*' without any parameters afterwards." + err = "Function 'bar' specifies '*' without following parameters." for block in dataset: with self.subTest(block=block): self.expect_failure(block, err) @@ -1679,7 +1679,7 @@ def test_depr_star_invalid_format_1(self): Docstring. """ err = ( - "Function 'foo.bar': expected format '* [from major.minor]' " + "Function 'bar': expected format '[from major.minor]' " "where 'major' and 'minor' are integers; got '3'" ) self.expect_failure(block, err, lineno=3) @@ -1693,7 +1693,7 @@ def test_depr_star_invalid_format_2(self): Docstring. """ err = ( - "Function 'foo.bar': expected format '* [from major.minor]' " + "Function 'bar': expected format '[from major.minor]' " "where 'major' and 'minor' are integers; got 'a.b'" ) self.expect_failure(block, err, lineno=3) @@ -1707,7 +1707,7 @@ def test_depr_star_invalid_format_3(self): Docstring. """ err = ( - "Function 'foo.bar': expected format '* [from major.minor]' " + "Function 'bar': expected format '[from major.minor]' " "where 'major' and 'minor' are integers; got '1.2.3'" ) self.expect_failure(block, err, lineno=3) @@ -1721,8 +1721,24 @@ def test_parameters_required_after_depr_star(self): Docstring. """ err = ( - "Function 'foo.bar' specifies '* [from ...]' without " - "any parameters afterwards" + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_parameters_required_after_depr_star2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + * + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." ) self.expect_failure(block, err, lineno=4) @@ -1735,7 +1751,7 @@ def test_depr_star_must_come_before_star(self): * [from 3.14] Docstring. """ - err = "Function 'foo.bar': '* [from ...]' must come before '*'" + err = "Function 'bar': '* [from ...]' must come before '*'" self.expect_failure(block, err, lineno=4) def test_depr_star_duplicate(self): @@ -1749,7 +1765,49 @@ def test_depr_star_duplicate(self): c: int Docstring. """ - err = "Function 'foo.bar' uses '[from ...]' more than once" + err = "Function 'bar' uses '* [from ...]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_star_duplicate2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.15] + c: int + Docstring. + """ + err = "Function 'bar' uses '* [from ...]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + b: int + / [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar' uses '/ [from ...]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate2(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + b: int + / [from 3.15] + c: int + Docstring. + """ + err = "Function 'bar' uses '/ [from ...]' more than once." self.expect_failure(block, err, lineno=5) def test_single_slash(self): @@ -1765,6 +1823,34 @@ def test_single_slash(self): ) self.expect_failure(block, err) + def test_parameters_required_before_depr_slash(self): + block = """ + module foo + foo.bar + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=2) + + def test_parameters_required_before_depr_slash2(self): + block = """ + module foo + foo.bar + a: int + / + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=4) + def test_double_slash(self): block = """ module foo @@ -1787,12 +1873,61 @@ def test_mix_star_and_slash(self): z: int / """ - err = ( - "Function 'bar' mixes keyword-only and positional-only parameters, " - "which is unsupported." - ) + err = "Function 'bar': '/' must precede '*'" self.expect_failure(block, err) + def test_depr_star_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_slash_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '/ [from ...]'" + self.expect_failure(block, err, lineno=4) + def test_parameters_not_permitted_after_slash_for_now(self): block = """ module foo @@ -2589,11 +2724,33 @@ class ClinicFunctionalTest(unittest.TestCase): locals().update((name, getattr(ac_tester, name)) for name in dir(ac_tester) if name.startswith('test_')) - def check_depr_star(self, pnames, fn, *args, **kwds): + def check_depr_star(self, pnames, fn, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' regex = ( fr"Passing( more than)?( [0-9]+)? positional argument(s)? to " - fr"{fn.__name__}\(\) is deprecated. Parameter(s)? {pnames} will " - fr"become( a)? keyword-only parameter(s)? in Python 3\.14" + fr"{re.escape(name)}\(\) is deprecated. Parameters? {pnames} will " + fr"become( a)? keyword-only parameters? in Python 3\.14" + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + + def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' + pl = 's' if ' ' in pnames else '' + regex = ( + fr"Passing keyword argument{pl} {pnames} to " + fr"{re.escape(name)}\(\) is deprecated. Corresponding parameter{pl} " + fr"will become positional-only in Python 3\.14." ) with self.assertWarnsRegex(DeprecationWarning, regex) as cm: # Record the line number, so we're sure we've got the correct stack @@ -3067,46 +3224,67 @@ def test_cloned_func_with_converter_exception_message(self): self.assertEqual(func(), name) def test_depr_star_new(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarNew() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - ac_tester.DeprStarNew(None) - self.assertEqual(cm.filename, __file__) + cls = ac_tester.DeprStarNew + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) def test_depr_star_new_cloned(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarNew.cloned() " - "is deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - obj = ac_tester.DeprStarNew(a=None) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - obj.cloned(None) - self.assertEqual(cm.filename, __file__) + fn = ac_tester.DeprStarNew().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarNew.cloned') def test_depr_star_init(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarInit() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - ac_tester.DeprStarInit(None) - self.assertEqual(cm.filename, __file__) + cls = ac_tester.DeprStarInit + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) def test_depr_star_init_cloned(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarInit.cloned() " - "is deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - obj = ac_tester.DeprStarInit(a=None) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - obj.cloned(None) - self.assertEqual(cm.filename, __file__) + fn = ac_tester.DeprStarInit().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarInit.cloned') + + def test_depr_star_init_noinline(self): + cls = ac_tester.DeprStarInitNoInline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") + + def test_depr_kwd_new(self): + cls = ac_tester.DeprKwdNew + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init(self): + cls = ac_tester.DeprKwdInit + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init_noinline(self): + cls = ac_tester.DeprKwdInitNoInline + cls = ac_tester.depr_star_noinline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") def test_depr_star_pos0_len1(self): fn = ac_tester.depr_star_pos0_len1 @@ -3177,6 +3355,103 @@ def test_depr_star_pos2_len2_with_kwd(self): check("a", "b", "c", d=0, e=0) check("a", "b", "c", "d", e=0) + def test_depr_star_noinline(self): + fn = ac_tester.depr_star_noinline + self.assertRaises(TypeError, fn, "a") + fn(a="a", b="b") + fn(a="a", b="b", c="c") + fn("a", b="b") + fn("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + + def test_depr_kwd_required_1(self): + fn = ac_tester.depr_kwd_required_1 + fn("a", "b") + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_required_2(self): + fn = ac_tester.depr_kwd_required_2 + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_1(self): + fn = ac_tester.depr_kwd_optional_1 + fn("a") + fn("a", "b") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_optional_2(self): + fn = ac_tester.depr_kwd_optional_2 + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", c="c") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_3(self): + fn = ac_tester.depr_kwd_optional_3 + fn() + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'a', 'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b") + check(a="a") + + def test_depr_kwd_required_optional(self): + fn = ac_tester.depr_kwd_required_optional + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_noinline(self): + fn = ac_tester.depr_kwd_noinline + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst new file mode 100644 index 00000000000000..ffdcfa6a429e64 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst @@ -0,0 +1,4 @@ +It is now possible to deprecate passing keyword arguments for +keyword-or-positional parameters with Argument Clinic, using the new ``/ +[from X.Y]`` syntax. (To be read as *"positional-only from Python version +X.Y"*.) See :ref:`clinic-howto-deprecate-keyword` for more information. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index af98d61ea7cccc..fe2196d0f7ee54 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -16,6 +16,17 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int cache_size, int uri, enum autocommit_mode autocommit); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.__init__'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.__init__'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.__init__'." +# endif +#endif + static int pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -59,28 +70,6 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) int uri = 0; enum autocommit_mode autocommit = LEGACY_TRANSACTION_CONTROL; - // Emit compiler warnings when we get to Python 3.15. - #if PY_VERSION_HEX >= 0x030f00C0 - # error \ - "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ - "'isolation_level', 'check_same_thread', 'factory', " \ - "'cached_statements' and 'uri' in the clinic input of " \ - "'_sqlite3.Connection.__init__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030f00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ - "'isolation_level', 'check_same_thread', 'factory', " \ - "'cached_statements' and 'uri' in the clinic input of " \ - "'_sqlite3.Connection.__init__' to be keyword-only.") - # else - # warning \ - "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ - "'isolation_level', 'check_same_thread', 'factory', " \ - "'cached_statements' and 'uri' in the clinic input of " \ - "'_sqlite3.Connection.__init__' to be keyword-only." - # endif - #endif if (nargs > 1 && nargs <= 8) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to _sqlite3.Connection()" @@ -89,7 +78,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) "'cached_statements' and 'uri' will become keyword-only " "parameters in Python 3.15.", 1)) { - goto exit; + goto exit; } } fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 8, 0, argsbuf); @@ -1692,4 +1681,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=5a05e5294ad9d2ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ad9d55977a51b8f input=a9049054013a1b77]*/ diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index c33536234af0bc..efec04d99029bb 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1195,14 +1195,14 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) /*[clinic input] output push -destination deprstar new file '{dirname}/clinic/_testclinic_depr_star.c.h' +destination deprstar new file '{dirname}/clinic/_testclinic_depr.c.h' output everything deprstar #output methoddef_ifndef buffer 1 output docstring_prototype suppress output parser_prototype suppress output impl_definition block [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f88f37038e00fb0a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=32116eac48a42d34]*/ // Mock Python version 3.8 @@ -1211,7 +1211,7 @@ output impl_definition block #define PY_VERSION_HEX 0x03080000 -#include "clinic/_testclinic_depr_star.c.h" +#include "clinic/_testclinic_depr.c.h" /*[clinic input] @@ -1219,13 +1219,13 @@ class _testclinic.DeprStarNew "PyObject *" "PyObject" @classmethod _testclinic.DeprStarNew.__new__ as depr_star_new * [from 3.14] - a: object + a: object = None The deprecation message should use the class name instead of __new__. [clinic start generated code]*/ static PyObject * depr_star_new_impl(PyTypeObject *type, PyObject *a) -/*[clinic end generated code: output=bdbb36244f90cf46 input=f4ae7dafbc23c378]*/ +/*[clinic end generated code: output=bdbb36244f90cf46 input=fdd640db964b4dc1]*/ { return type->tp_alloc(type, 0); } @@ -1260,13 +1260,13 @@ static PyTypeObject DeprStarNew = { class _testclinic.DeprStarInit "PyObject *" "PyObject" _testclinic.DeprStarInit.__init__ as depr_star_init * [from 3.14] - a: object + a: object = None The deprecation message should use the class name instead of __init__. [clinic start generated code]*/ static int depr_star_init_impl(PyObject *self, PyObject *a) -/*[clinic end generated code: output=8d27b43c286d3ecc input=659ebc748d87fa86]*/ +/*[clinic end generated code: output=8d27b43c286d3ecc input=5575b77229d5e2be]*/ { return 0; } @@ -1298,6 +1298,116 @@ static PyTypeObject DeprStarInit = { }; +/*[clinic input] +class _testclinic.DeprStarInitNoInline "PyObject *" "PyObject" +_testclinic.DeprStarInitNoInline.__init__ as depr_star_init_noinline + a: object + * [from 3.14] + b: object + c: object = None + * + # Force to use _PyArg_ParseTupleAndKeywordsFast. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static int +depr_star_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=9b31fc167f1bf9f7 input=5a887543122bca48]*/ +{ + return 0; +} + +static PyTypeObject DeprStarInitNoInline = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprStarInitNoInline", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_star_init_noinline, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprKwdNew "PyObject *" "PyObject" +@classmethod +_testclinic.DeprKwdNew.__new__ as depr_kwd_new + a: object = None + / [from 3.14] +The deprecation message should use the class name instead of __new__. +[clinic start generated code]*/ + +static PyObject * +depr_kwd_new_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=618d07afc5616149 input=6c7d13c471013c10]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject DeprKwdNew = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprKwdNew", + .tp_basicsize = sizeof(PyObject), + .tp_new = depr_kwd_new, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprKwdInit "PyObject *" "PyObject" +_testclinic.DeprKwdInit.__init__ as depr_kwd_init + a: object = None + / [from 3.14] +The deprecation message should use the class name instead of __init__. +[clinic start generated code]*/ + +static int +depr_kwd_init_impl(PyObject *self, PyObject *a) +/*[clinic end generated code: output=6e02eb724a85d840 input=b9bf3c20f012d539]*/ +{ + return 0; +} + +static PyTypeObject DeprKwdInit = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprKwdInit", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_kwd_init, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprKwdInitNoInline "PyObject *" "PyObject" +_testclinic.DeprKwdInitNoInline.__init__ as depr_kwd_init_noinline + a: object + / + b: object + c: object = None + / [from 3.14] + # Force to use _PyArg_ParseTupleAndKeywordsFast. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static int +depr_kwd_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=27759d70ddd25873 input=c19d982c8c70a930]*/ +{ + return 0; +} + +static PyTypeObject DeprKwdInitNoInline = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprKwdInitNoInline", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_kwd_init_noinline, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + /*[clinic input] depr_star_pos0_len1 * [from 3.14] @@ -1450,6 +1560,148 @@ depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, } +/*[clinic input] +depr_star_noinline + a: object + * [from 3.14] + b: object + c: object = None + * + # Force to use _PyArg_ParseStackAndKeywords. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static PyObject * +depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=cc27dacf5c2754af input=d36cc862a2daef98]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_required_1 + a: object + / + b: object + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_required_1_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=1d8ab19ea78418af input=53f2c398b828462d]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_required_2 + a: object + / + b: object + c: object + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_required_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=44a89cb82509ddde input=a2b0ef37de8a01a7]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_optional_1 + a: object + / + b: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_optional_1_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=a8a3d67efcc7b058 input=e416981eb78c3053]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_optional_2 + a: object + / + b: object = None + c: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_optional_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=aa2d967f26fdb9f6 input=cae3afb783bfc855]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_optional_3 + a: object = None + b: object = None + c: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_optional_3_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=a26025bf6118fd07 input=c9183b2f9ccaf992]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_required_optional + a: object + / + b: object + c: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_required_optional_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=e53a8b7a250d8ffc input=23237a046f8388f5]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_noinline + a: object + / + b: object + c: object = None + / [from 3.14] + # Force to use _PyArg_ParseStackAndKeywords. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static PyObject * +depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=f59da8113f2bad7c input=1d6db65bebb069d7]*/ +{ + Py_RETURN_NONE; +} + // Reset PY_VERSION_HEX #undef PY_VERSION_HEX #define PY_VERSION_HEX _SAVED_PY_VERSION @@ -1526,6 +1778,14 @@ static PyMethodDef tester_methods[] = { DEPR_STAR_POS2_LEN1_METHODDEF DEPR_STAR_POS2_LEN2_METHODDEF DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF + DEPR_STAR_NOINLINE_METHODDEF + DEPR_KWD_REQUIRED_1_METHODDEF + DEPR_KWD_REQUIRED_2_METHODDEF + DEPR_KWD_OPTIONAL_1_METHODDEF + DEPR_KWD_OPTIONAL_2_METHODDEF + DEPR_KWD_OPTIONAL_3_METHODDEF + DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF + DEPR_KWD_NOINLINE_METHODDEF {NULL, NULL} }; @@ -1549,6 +1809,18 @@ PyInit__testclinic(void) if (PyModule_AddType(m, &DeprStarInit) < 0) { goto error; } + if (PyModule_AddType(m, &DeprStarInitNoInline) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprKwdNew) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprKwdInit) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprKwdInitNoInline) < 0) { + goto error; + } return m; error: diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h new file mode 100644 index 00000000000000..661fdaf2a07181 --- /dev/null +++ b/Modules/clinic/_testclinic_depr.c.h @@ -0,0 +1,2095 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(depr_star_new__doc__, +"DeprStarNew(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static PyObject * +depr_star_new_impl(PyTypeObject *type, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarNew.__new__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarNew.__new__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarNew.__new__'." +# endif +#endif + +static PyObject * +depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_star_new_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_new_clone__doc__, +"cloned($self, /, a=None)\n" +"--\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" +"is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_NEW_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_new_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_new_clone__doc__}, + +static PyObject * +depr_star_new_clone_impl(PyObject *type, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarNew.cloned'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarNew.cloned'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarNew.cloned'." +# endif +#endif + +static PyObject * +depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew.cloned()" + " is deprecated. Parameter 'a' will become a keyword-only " + "parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = args[0]; +skip_optional_pos: + return_value = depr_star_new_clone_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_init__doc__, +"DeprStarInit(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static int +depr_star_init_impl(PyObject *self, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarInit.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarInit.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarInit.__init__'." +# endif +#endif + +static int +depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarInit() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_star_init_impl(self, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_init_clone__doc__, +"cloned($self, /, a=None)\n" +"--\n" +"\n" +"Note: Passing positional arguments to\n" +"_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" +"become a keyword-only parameter in Python 3.14.\n" +""); + +#define DEPR_STAR_INIT_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_init_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_init_clone__doc__}, + +static PyObject * +depr_star_init_clone_impl(PyObject *self, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarInit.cloned'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarInit.cloned'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarInit.cloned'." +# endif +#endif + +static PyObject * +depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " + "will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = args[0]; +skip_optional_pos: + return_value = depr_star_init_clone_impl(self, a); + +exit: + return return_value; +} + +static int +depr_star_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'." +# endif +#endif + +static int +depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|O$s#:DeprStarInitNoInline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "_testclinic.DeprStarInitNoInline() is deprecated. Parameters 'b'" + " and 'c' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + return_value = depr_star_init_noinline_impl(self, a, b, c, d, d_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_new__doc__, +"DeprKwdNew(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing keyword argument \'a\' to _testclinic.DeprKwdNew() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +static PyObject * +depr_kwd_new_impl(PyTypeObject *type, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprKwdNew.__new__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprKwdNew.__new__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprKwdNew.__new__'." +# endif +#endif + +static PyObject * +depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprKwdNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'a' to _testclinic.DeprKwdNew() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_kwd_new_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_init__doc__, +"DeprKwdInit(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing keyword argument \'a\' to _testclinic.DeprKwdInit() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +static int +depr_kwd_init_impl(PyObject *self, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprKwdInit.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprKwdInit.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprKwdInit.__init__'." +# endif +#endif + +static int +depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprKwdInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'a' to _testclinic.DeprKwdInit() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_kwd_init_impl(self, a); + +exit: + return return_value; +} + +static int +depr_kwd_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'." +# endif +#endif + +static int +depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|Os#:DeprKwdInitNoInline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, &_Py_ID(c))))) { + if (PyErr_Occurred()) { // PyDict_Contains() above can fail + goto exit; + } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to " + "_testclinic.DeprKwdInitNoInline() is deprecated. Corresponding " + "parameters will become positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + return_value = depr_kwd_init_noinline_impl(self, a, b, c, d, d_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len1__doc__, +"depr_star_pos0_len1($module, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len1() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN1_METHODDEF \ + {"depr_star_pos0_len1", _PyCFunction_CAST(depr_star_pos0_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len1__doc__}, + +static PyObject * +depr_star_pos0_len1_impl(PyObject *module, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos0_len1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos0_len1'.") +# else +# warning "Update the clinic input of 'depr_star_pos0_len1'." +# endif +#endif + +static PyObject * +depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len1() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_pos0_len1_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len2__doc__, +"depr_star_pos0_len2($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len2() is\n" +"deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" +"in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN2_METHODDEF \ + {"depr_star_pos0_len2", _PyCFunction_CAST(depr_star_pos0_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len2__doc__}, + +static PyObject * +depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos0_len2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos0_len2'.") +# else +# warning "Update the clinic input of 'depr_star_pos0_len2'." +# endif +#endif + +static PyObject * +depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + if (nargs > 0 && nargs <= 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len2() is " + "deprecated. Parameters 'a' and 'b' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos0_len2_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, +"depr_star_pos0_len3_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" +"is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" +"parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ + {"depr_star_pos0_len3_with_kwd", _PyCFunction_CAST(depr_star_pos0_len3_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len3_with_kwd__doc__}, + +static PyObject * +depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos0_len3_with_kwd'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos0_len3_with_kwd'.") +# else +# warning "Update the clinic input of 'depr_star_pos0_len3_with_kwd'." +# endif +#endif + +static PyObject * +depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len3_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + if (nargs > 0 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len3_with_kwd() " + "is deprecated. Parameters 'a', 'b' and 'c' will become " + "keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos0_len3_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, +"depr_star_pos1_len1_opt($module, /, a, b=None)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ + {"depr_star_pos1_len1_opt", _PyCFunction_CAST(depr_star_pos1_len1_opt), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1_opt__doc__}, + +static PyObject * +depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos1_len1_opt'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos1_len1_opt'.") +# else +# warning "Update the clinic input of 'depr_star_pos1_len1_opt'." +# endif +#endif + +static PyObject * +depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = depr_star_pos1_len1_opt_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1__doc__, +"depr_star_pos1_len1($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_METHODDEF \ + {"depr_star_pos1_len1", _PyCFunction_CAST(depr_star_pos1_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1__doc__}, + +static PyObject * +depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos1_len1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos1_len1'.") +# else +# warning "Update the clinic input of 'depr_star_pos1_len1'." +# endif +#endif + +static PyObject * +depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos1_len1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, +"depr_star_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to\n" +"depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos1_len2_with_kwd", _PyCFunction_CAST(depr_star_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos1_len2_with_kwd'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos1_len2_with_kwd'.") +# else +# warning "Update the clinic input of 'depr_star_pos1_len2_with_kwd'." +# endif +#endif + +static PyObject * +depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" + " 'c' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos1_len2_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len1__doc__, +"depr_star_pos2_len1($module, /, a, b, c)\n" +"--\n" +"\n" +"Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" +"deprecated. Parameter \'c\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN1_METHODDEF \ + {"depr_star_pos2_len1", _PyCFunction_CAST(depr_star_pos2_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len1__doc__}, + +static PyObject * +depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos2_len1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos2_len1'.") +# else +# warning "Update the clinic input of 'depr_star_pos2_len1'." +# endif +#endif + +static PyObject * +depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + if (nargs == 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 3 positional arguments to depr_star_pos2_len1() is " + "deprecated. Parameter 'c' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = depr_star_pos2_len1_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2__doc__, +"depr_star_pos2_len2($module, /, a, b, c, d)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" +"become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_METHODDEF \ + {"depr_star_pos2_len2", _PyCFunction_CAST(depr_star_pos2_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2__doc__}, + +static PyObject * +depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos2_len2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos2_len2'.") +# else +# warning "Update the clinic input of 'depr_star_pos2_len2'." +# endif +#endif + +static PyObject * +depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" + " become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos2_len2_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, +"depr_star_pos2_len2_with_kwd($module, /, a, b, c, d, *, e)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos2_len2_with_kwd", _PyCFunction_CAST(depr_star_pos2_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos2_len2_with_kwd'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos2_len2_with_kwd'.") +# else +# warning "Update the clinic input of 'depr_star_pos2_len2_with_kwd'." +# endif +#endif + +static PyObject * +depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" + " 'd' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + return_value = depr_star_pos2_len2_with_kwd_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_noinline__doc__, +"depr_star_noinline($module, /, a, b, c=None, *, d=\'\')\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to depr_star_noinline()\n" +"is deprecated. Parameters \'b\' and \'c\' will become keyword-only\n" +"parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_NOINLINE_METHODDEF \ + {"depr_star_noinline", _PyCFunction_CAST(depr_star_noinline), METH_FASTCALL|METH_KEYWORDS, depr_star_noinline__doc__}, + +static PyObject * +depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_noinline'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_noinline'.") +# else +# warning "Update the clinic input of 'depr_star_noinline'." +# endif +#endif + +static PyObject * +depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|O$s#:depr_star_noinline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to depr_star_noinline() " + "is deprecated. Parameters 'b' and 'c' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; + } + } + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + return_value = depr_star_noinline_impl(module, a, b, c, d, d_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_required_1__doc__, +"depr_kwd_required_1($module, a, /, b)\n" +"--\n" +"\n" +"Note: Passing keyword argument \'b\' to depr_kwd_required_1() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_REQUIRED_1_METHODDEF \ + {"depr_kwd_required_1", _PyCFunction_CAST(depr_kwd_required_1), METH_FASTCALL|METH_KEYWORDS, depr_kwd_required_1__doc__}, + +static PyObject * +depr_kwd_required_1_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_required_1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_required_1'.") +# else +# warning "Update the clinic input of 'depr_kwd_required_1'." +# endif +#endif + +static PyObject * +depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_required_1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'b' to depr_kwd_required_1() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + return_value = depr_kwd_required_1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_required_2__doc__, +"depr_kwd_required_2($module, a, /, b, c)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_required_2()\n" +"is deprecated. Corresponding parameters will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_REQUIRED_2_METHODDEF \ + {"depr_kwd_required_2", _PyCFunction_CAST(depr_kwd_required_2), METH_FASTCALL|METH_KEYWORDS, depr_kwd_required_2__doc__}, + +static PyObject * +depr_kwd_required_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_required_2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_required_2'.") +# else +# warning "Update the clinic input of 'depr_kwd_required_2'." +# endif +#endif + +static PyObject * +depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_required_2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_kwd_required_2() " + "is deprecated. Corresponding parameters will become " + "positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = depr_kwd_required_2_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_optional_1__doc__, +"depr_kwd_optional_1($module, a, /, b=None)\n" +"--\n" +"\n" +"Note: Passing keyword argument \'b\' to depr_kwd_optional_1() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_OPTIONAL_1_METHODDEF \ + {"depr_kwd_optional_1", _PyCFunction_CAST(depr_kwd_optional_1), METH_FASTCALL|METH_KEYWORDS, depr_kwd_optional_1__doc__}, + +static PyObject * +depr_kwd_optional_1_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_optional_1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_optional_1'.") +# else +# warning "Update the clinic input of 'depr_kwd_optional_1'." +# endif +#endif + +static PyObject * +depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_optional_1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && nargs < 2 && args[1]) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'b' to depr_kwd_optional_1() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = depr_kwd_optional_1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_optional_2__doc__, +"depr_kwd_optional_2($module, a, /, b=None, c=None)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_optional_2()\n" +"is deprecated. Corresponding parameters will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_OPTIONAL_2_METHODDEF \ + {"depr_kwd_optional_2", _PyCFunction_CAST(depr_kwd_optional_2), METH_FASTCALL|METH_KEYWORDS, depr_kwd_optional_2__doc__}, + +static PyObject * +depr_kwd_optional_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_optional_2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_optional_2'.") +# else +# warning "Update the clinic input of 'depr_kwd_optional_2'." +# endif +#endif + +static PyObject * +depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_optional_2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_kwd_optional_2() " + "is deprecated. Corresponding parameters will become " + "positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = depr_kwd_optional_2_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_optional_3__doc__, +"depr_kwd_optional_3($module, /, a=None, b=None, c=None)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'a\', \'b\' and \'c\' to\n" +"depr_kwd_optional_3() is deprecated. Corresponding parameters will\n" +"become positional-only in Python 3.14.\n" +""); + +#define DEPR_KWD_OPTIONAL_3_METHODDEF \ + {"depr_kwd_optional_3", _PyCFunction_CAST(depr_kwd_optional_3), METH_FASTCALL|METH_KEYWORDS, depr_kwd_optional_3__doc__}, + +static PyObject * +depr_kwd_optional_3_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_optional_3'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_optional_3'.") +# else +# warning "Update the clinic input of 'depr_kwd_optional_3'." +# endif +#endif + +static PyObject * +depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_optional_3", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *a = Py_None; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 1 && args[0]) || (nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'a', 'b' and 'c' to " + "depr_kwd_optional_3() is deprecated. Corresponding parameters " + "will become positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + a = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = depr_kwd_optional_3_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_required_optional__doc__, +"depr_kwd_required_optional($module, a, /, b, c=None)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to\n" +"depr_kwd_required_optional() is deprecated. Corresponding parameters\n" +"will become positional-only in Python 3.14.\n" +""); + +#define DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF \ + {"depr_kwd_required_optional", _PyCFunction_CAST(depr_kwd_required_optional), METH_FASTCALL|METH_KEYWORDS, depr_kwd_required_optional__doc__}, + +static PyObject * +depr_kwd_required_optional_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_required_optional'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_required_optional'.") +# else +# warning "Update the clinic input of 'depr_kwd_required_optional'." +# endif +#endif + +static PyObject * +depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_required_optional", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && args[2]))) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to " + "depr_kwd_required_optional() is deprecated. Corresponding " + "parameters will become positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + c = args[2]; +skip_optional_pos: + return_value = depr_kwd_required_optional_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_noinline__doc__, +"depr_kwd_noinline($module, a, /, b, c=None, d=\'\')\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_noinline() is\n" +"deprecated. Corresponding parameters will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_NOINLINE_METHODDEF \ + {"depr_kwd_noinline", _PyCFunction_CAST(depr_kwd_noinline), METH_FASTCALL|METH_KEYWORDS, depr_kwd_noinline__doc__}, + +static PyObject * +depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_noinline'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_noinline'.") +# else +# warning "Update the clinic input of 'depr_kwd_noinline'." +# endif +#endif + +static PyObject * +depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|Os#:depr_kwd_noinline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, &_Py_ID(c))))) { + if (PyErr_Occurred()) { // PySequence_Contains() above can fail + goto exit; + } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_kwd_noinline() is " + "deprecated. Corresponding parameters will become positional-only" + " in Python 3.14.", 1)) + { + goto exit; + } + } + return_value = depr_kwd_noinline_impl(module, a, b, c, d, d_length); + +exit: + return return_value; +} +/*[clinic end generated code: output=fc558c1efdcab076 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr_star.c.h b/Modules/clinic/_testclinic_depr_star.c.h deleted file mode 100644 index 1aa42dd4059777..00000000000000 --- a/Modules/clinic/_testclinic_depr_star.c.h +++ /dev/null @@ -1,1140 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -PyDoc_STRVAR(depr_star_new__doc__, -"DeprStarNew(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __new__.\n" -"\n" -"Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" -"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -static PyObject * -depr_star_new_impl(PyTypeObject *type, PyObject *a); - -static PyObject * -depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "DeprStarNew", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.__new__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.__new__' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.__new__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to _testclinic.DeprStarNew() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = depr_star_new_impl(type, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_new_clone__doc__, -"cloned($self, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" -"is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_NEW_CLONE_METHODDEF \ - {"cloned", _PyCFunction_CAST(depr_star_new_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_new_clone__doc__}, - -static PyObject * -depr_star_new_clone_impl(PyObject *type, PyObject *a); - -static PyObject * -depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "cloned", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.cloned' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.cloned' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.cloned' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to _testclinic.DeprStarNew.cloned()" - " is deprecated. Parameter 'a' will become a keyword-only " - "parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = depr_star_new_clone_impl(type, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_init__doc__, -"DeprStarInit(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __init__.\n" -"\n" -"Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" -"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -static int -depr_star_init_impl(PyObject *self, PyObject *a); - -static int -depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "DeprStarInit", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.__init__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.__init__' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.__init__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to _testclinic.DeprStarInit() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = depr_star_init_impl(self, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_init_clone__doc__, -"cloned($self, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" -"become a keyword-only parameter in Python 3.14.\n" -""); - -#define DEPR_STAR_INIT_CLONE_METHODDEF \ - {"cloned", _PyCFunction_CAST(depr_star_init_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_init_clone__doc__}, - -static PyObject * -depr_star_init_clone_impl(PyObject *self, PyObject *a); - -static PyObject * -depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "cloned", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.cloned' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.cloned' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.cloned' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " - "will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = depr_star_init_clone_impl(self, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos0_len1__doc__, -"depr_star_pos0_len1($module, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to depr_star_pos0_len1() is\n" -"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS0_LEN1_METHODDEF \ - {"depr_star_pos0_len1", _PyCFunction_CAST(depr_star_pos0_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len1__doc__}, - -static PyObject * -depr_star_pos0_len1_impl(PyObject *module, PyObject *a); - -static PyObject * -depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos0_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " 'depr_star_pos0_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " 'depr_star_pos0_len1' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " 'depr_star_pos0_len1' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to depr_star_pos0_len1() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = depr_star_pos0_len1_impl(module, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos0_len2__doc__, -"depr_star_pos0_len2($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing positional arguments to depr_star_pos0_len2() is\n" -"deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" -"in Python 3.14.\n" -""); - -#define DEPR_STAR_POS0_LEN2_METHODDEF \ - {"depr_star_pos0_len2", _PyCFunction_CAST(depr_star_pos0_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len2__doc__}, - -static PyObject * -depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); - -static PyObject * -depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos0_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'depr_star_pos0_len2' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'depr_star_pos0_len2' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'depr_star_pos0_len2' to be keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to depr_star_pos0_len2() is " - "deprecated. Parameters 'a' and 'b' will become keyword-only " - "parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = depr_star_pos0_len2_impl(module, a, b); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, -"depr_star_pos0_len3_with_kwd($module, /, a, b, c, *, d)\n" -"--\n" -"\n" -"Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" -"is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" -"parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ - {"depr_star_pos0_len3_with_kwd", _PyCFunction_CAST(depr_star_pos0_len3_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len3_with_kwd__doc__}, - -static PyObject * -depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos0_len3_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ - "keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to depr_star_pos0_len3_with_kwd() " - "is deprecated. Parameters 'a', 'b' and 'c' will become " - "keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = depr_star_pos0_len3_with_kwd_impl(module, a, b, c, d); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, -"depr_star_pos1_len1_opt($module, /, a, b=None)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" -"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ - {"depr_star_pos1_len1_opt", _PyCFunction_CAST(depr_star_pos1_len1_opt), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1_opt__doc__}, - -static PyObject * -depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); - -static PyObject * -depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos1_len1_opt", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *a; - PyObject *b = Py_None; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1_opt' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1_opt' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1_opt' to be keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " - "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - if (!noptargs) { - goto skip_optional_pos; - } - b = args[1]; -skip_optional_pos: - return_value = depr_star_pos1_len1_opt_impl(module, a, b); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos1_len1__doc__, -"depr_star_pos1_len1($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" -"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS1_LEN1_METHODDEF \ - {"depr_star_pos1_len1", _PyCFunction_CAST(depr_star_pos1_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1__doc__}, - -static PyObject * -depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); - -static PyObject * -depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos1_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1' to be keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to depr_star_pos1_len1() is " - "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = depr_star_pos1_len1_impl(module, a, b); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, -"depr_star_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" -"--\n" -"\n" -"Note: Passing more than 1 positional argument to\n" -"depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" -"will become keyword-only parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ - {"depr_star_pos1_len2_with_kwd", _PyCFunction_CAST(depr_star_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len2_with_kwd__doc__}, - -static PyObject * -depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos1_len2_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." - # endif - #endif - if (nargs > 1 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to " - "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" - " 'c' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = depr_star_pos1_len2_with_kwd_impl(module, a, b, c, d); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos2_len1__doc__, -"depr_star_pos2_len1($module, /, a, b, c)\n" -"--\n" -"\n" -"Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" -"deprecated. Parameter \'c\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS2_LEN1_METHODDEF \ - {"depr_star_pos2_len1", _PyCFunction_CAST(depr_star_pos2_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len1__doc__}, - -static PyObject * -depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c); - -static PyObject * -depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos2_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - PyObject *a; - PyObject *b; - PyObject *c; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ - " 'depr_star_pos2_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ - " 'depr_star_pos2_len1' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ - " 'depr_star_pos2_len1' to be keyword-only." - # endif - #endif - if (nargs == 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 3 positional arguments to depr_star_pos2_len1() is " - "deprecated. Parameter 'c' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - return_value = depr_star_pos2_len1_impl(module, a, b, c); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos2_len2__doc__, -"depr_star_pos2_len2($module, /, a, b, c, d)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" -"become keyword-only parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS2_LEN2_METHODDEF \ - {"depr_star_pos2_len2", _PyCFunction_CAST(depr_star_pos2_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2__doc__}, - -static PyObject * -depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos2_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2' to be keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" - " become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = depr_star_pos2_len2_impl(module, a, b, c, d); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, -"depr_star_pos2_len2_with_kwd($module, /, a, b, c, d, *, e)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" -"will become keyword-only parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ - {"depr_star_pos2_len2_with_kwd", _PyCFunction_CAST(depr_star_pos2_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2_with_kwd__doc__}, - -static PyObject * -depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d, PyObject *e); - -static PyObject * -depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 5 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos2_len2_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[5]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - PyObject *e; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" - " 'd' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - e = args[4]; - return_value = depr_star_pos2_len2_with_kwd_impl(module, a, b, c, d, e); - -exit: - return return_value; -} -/*[clinic end generated code: output=7a16fee4d6742d54 input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 28bd2a4430d14e..58f9c8e8a3e9a5 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -323,7 +323,11 @@ Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - Modules/_testcapi/vectorcall.c - MethodDescriptor2_Type - Modules/_testclinic.c - DeprStarInit - +Modules/_testclinic.c - DeprStarInitNoInline - Modules/_testclinic.c - DeprStarNew - +Modules/_testclinic.c - DeprKwdInit - +Modules/_testclinic.c - DeprKwdInitNoInline - +Modules/_testclinic.c - DeprKwdNew - ################################## diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1593dc49e07e1f..fe84e810760b40 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -849,25 +849,24 @@ class CLanguage(Language): #define {methoddef_name} #endif /* !defined({methoddef_name}) */ """) - DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" + COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" // Emit compiler warnings when we get to Python {major}.{minor}. #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error \ - {cpp_message} + # error {message} #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 # ifdef _MSC_VER - # pragma message ( \ - {cpp_message}) + # pragma message ({message}) # else - # warning \ - {cpp_message} + # warning {message} # endif #endif - if ({condition}) {{{{ + """ + DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + if ({condition}) {{{{{errcheck} if (PyErr_WarnEx(PyExc_DeprecationWarning, - {depr_message}, 1)) + {message}, 1)) {{{{ - goto exit; + goto exit; }}}} }}}} """ @@ -893,6 +892,30 @@ def render( function = o return self.render_function(clinic, function) + def compiler_deprecated_warning( + self, + func: Function, + parameters: list[Parameter], + ) -> str | None: + minversion: VersionTuple | None = None + for p in parameters: + for version in p.deprecated_positional, p.deprecated_keyword: + if version and (not minversion or minversion > version): + minversion = version + if not minversion: + return None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + source = os.path.basename(self.cpp.filename) + message = f"Update the clinic input of {func.full_name!r}." + code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( + major=minversion[0], + minor=minversion[1], + message=c_repr(message), + ) + return normalize_snippet(code) + def deprecate_positional_use( self, func: Function, @@ -910,15 +933,7 @@ def deprecate_positional_use( assert first_param.deprecated_positional == last_param.deprecated_positional thenceforth = first_param.deprecated_positional assert thenceforth is not None - - # Format the preprocessor warning and error messages. - assert isinstance(self.cpp.filename, str) - source = os.path.basename(self.cpp.filename) major, minor = thenceforth - cpp_message = ( - f"In {source}, update parameter(s) {pstr} in the clinic " - f"input of {func.full_name!r} to be keyword-only." - ) # Format the deprecation message. if first_pos == 0: @@ -927,7 +942,7 @@ def deprecate_positional_use( condition = f"nargs == {first_pos+1}" if first_pos: preamble = f"Passing {first_pos+1} positional arguments to " - depr_message = preamble + ( + message = preamble + ( f"{func.fulldisplayname}() is deprecated. Parameter {pstr} will " f"become a keyword-only parameter in Python {major}.{minor}." ) @@ -938,26 +953,93 @@ def deprecate_positional_use( f"Passing more than {first_pos} positional " f"argument{'s' if first_pos != 1 else ''} to " ) - depr_message = preamble + ( + message = preamble + ( f"{func.fulldisplayname}() is deprecated. Parameters {pstr} will " f"become keyword-only parameters in Python {major}.{minor}." ) # Append deprecation warning to docstring. - lines = textwrap.wrap(f"Note: {depr_message}") - docstring = "\n".join(lines) + docstring = textwrap.fill(f"Note: {message}") func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck="", + message=wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return normalize_snippet(code, indent=4) + + def deprecate_keyword_use( + self, + func: Function, + params: dict[int, Parameter], + argname_fmt: str | None, + ) -> str: + assert len(params) > 0 + names = [repr(p.name) for p in params.values()] + first_param = next(iter(params.values())) + last_param = next(reversed(params.values())) + + # Pretty-print list of names. + pstr = pprint_words(names) + + # For now, assume there's only one deprecation level. + assert first_param.deprecated_keyword == last_param.deprecated_keyword + thenceforth = first_param.deprecated_keyword + assert thenceforth is not None + major, minor = thenceforth + # Format the deprecation message. + containscheck = "" + conditions = [] + for i, p in params.items(): + if p.is_optional(): + if argname_fmt: + conditions.append(f"nargs < {i+1} && {argname_fmt % i}") + elif func.kind.new_or_init: + conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") + containscheck = "PyDict_Contains" + else: + conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") + containscheck = "PySequence_Contains" + else: + conditions = [f"nargs < {i+1}"] + condition = ") || (".join(conditions) + if len(conditions) > 1: + condition = f"(({condition}))" + if last_param.is_optional(): + if func.kind.new_or_init: + condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" + else: + condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" + if len(params) == 1: + what1 = "argument" + what2 = "parameter" + else: + what1 = "arguments" + what2 = "parameters" + message = ( + f"Passing keyword {what1} {pstr} to {func.fulldisplayname}() is deprecated. " + f"Corresponding {what2} will become positional-only in Python {major}.{minor}." + ) + if containscheck: + errcheck = f""" + if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail + goto exit; + }}}}""" + else: + errcheck = "" + if argname_fmt: + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" # Format and return the code block. - code = self.DEPRECATED_POSITIONAL_PROTOTYPE.format( + code = self.DEPRECATION_WARNING_PROTOTYPE.format( condition=condition, - major=major, - minor=minor, - cpp_message=wrapped_c_string_literal(cpp_message, suffix=" \\", - width=64, - subsequent_indent=16), - depr_message=wrapped_c_string_literal(depr_message, width=64, - subsequent_indent=20), + errcheck=errcheck, + message=wrapped_c_string_literal(message, width=64, + subsequent_indent=20), ) return normalize_snippet(code, indent=4) @@ -1258,6 +1340,14 @@ def parser_body( parser_definition = parser_body(parser_prototype, *parser_code) else: + deprecated_positionals: dict[int, Parameter] = {} + deprecated_keywords: dict[int, Parameter] = {} + for i, p in enumerate(parameters): + if p.deprecated_positional: + deprecated_positionals[i] = p + if p.deprecated_keyword: + deprecated_keywords[i] = p + has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) if vararg == NO_VARARG: args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( @@ -1310,7 +1400,10 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - deprecated_positionals: dict[int, Parameter] = {} + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt) + parser_code.append(code) + add_label: str | None = None for i, p in enumerate(parameters): if isinstance(p.converter, defining_class_converter): @@ -1325,8 +1418,6 @@ def parser_body( parser_code.append("%s:" % add_label) add_label = None if not p.is_optional(): - if p.deprecated_positional: - deprecated_positionals[i] = p parser_code.append(normalize_snippet(parsearg, indent=4)) elif i < pos_only: add_label = 'skip_optional_posonly' @@ -1356,8 +1447,6 @@ def parser_body( goto %s; }} """ % add_label, indent=4)) - if p.deprecated_positional: - deprecated_positionals[i] = p if i + 1 == len(parameters): parser_code.append(normalize_snippet(parsearg, indent=4)) else: @@ -1373,12 +1462,6 @@ def parser_body( }} """ % add_label, indent=4)) - if deprecated_positionals: - code = self.deprecate_positional_use(f, deprecated_positionals) - assert parser_code is not None - # Insert the deprecation code before parameter parsing. - parser_code.insert(0, code) - if parser_code is not None: if add_label: parser_code.append("%s:" % add_label) @@ -1398,6 +1481,17 @@ def parser_body( goto exit; }} """, indent=4)] + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, None) + parser_code.append(code) + + if deprecated_positionals: + code = self.deprecate_positional_use(f, deprecated_positionals) + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + parser_definition = parser_body(parser_prototype, *parser_code, declarations=declarations) @@ -1478,6 +1572,10 @@ def parser_body( parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) + compiler_warning = self.compiler_deprecated_warning(f, parameters) + if compiler_warning: + parser_definition = compiler_warning + "\n\n" + parser_definition + d = { "docstring_prototype" : docstring_prototype, "docstring_definition" : docstring_definition, @@ -2739,6 +2837,7 @@ class Parameter: group: int = 0 # (`None` signifies that there is no deprecation) deprecated_positional: VersionTuple | None = None + deprecated_keyword: VersionTuple | None = None right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: @@ -4576,6 +4675,7 @@ class DSLParser: keyword_only: bool positional_only: bool deprecated_positional: VersionTuple | None + deprecated_keyword: VersionTuple | None group: int parameter_state: ParamState indent: IndentStack @@ -4583,11 +4683,7 @@ class DSLParser: coexist: bool parameter_continuation: str preserve_output: bool - star_from_version_re = create_regex( - before="* [from ", - after="]", - word=False, - ) + from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') def __init__(self, clinic: Clinic) -> None: self.clinic = clinic @@ -4612,6 +4708,7 @@ def reset(self) -> None: self.keyword_only = False self.positional_only = False self.deprecated_positional = None + self.deprecated_keyword = None self.group = 0 self.parameter_state: ParamState = ParamState.START self.indent = IndentStack() @@ -5089,21 +5186,22 @@ def state_parameter(self, line: str) -> None: return line = line.lstrip() - match = self.star_from_version_re.match(line) + version: VersionTuple | None = None + match = self.from_version_re.fullmatch(line) if match: - self.parse_deprecated_positional(match.group(1)) - return + line = match[1] + version = self.parse_version(match[2]) func = self.function match line: case '*': - self.parse_star(func) + self.parse_star(func, version) case '[': self.parse_opening_square_bracket(func) case ']': self.parse_closing_square_bracket(func) case '/': - self.parse_slash(func) + self.parse_slash(func, version) case param: self.parse_parameter(param) @@ -5404,29 +5502,36 @@ def parse_converter( "Annotations must be either a name, a function call, or a string." ) - def parse_deprecated_positional(self, thenceforth: str) -> None: + def parse_version(self, thenceforth: str) -> VersionTuple: + """Parse Python version in `[from ...]` marker.""" assert isinstance(self.function, Function) - fname = self.function.full_name - if self.keyword_only: - fail(f"Function {fname!r}: '* [from ...]' must come before '*'") - if self.deprecated_positional: - fail(f"Function {fname!r} uses '[from ...]' more than once.") try: major, minor = thenceforth.split(".") - self.deprecated_positional = int(major), int(minor) + return int(major), int(minor) except ValueError: fail( - f"Function {fname!r}: expected format '* [from major.minor]' " + f"Function {self.function.name!r}: expected format '[from major.minor]' " f"where 'major' and 'minor' are integers; got {thenceforth!r}" ) - def parse_star(self, function: Function) -> None: - """Parse keyword-only parameter marker '*'.""" - if self.keyword_only: - fail(f"Function {function.name!r} uses '*' more than once.") - self.deprecated_positional = None - self.keyword_only = True + def parse_star(self, function: Function, version: VersionTuple | None) -> None: + """Parse keyword-only parameter marker '*'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.keyword_only: + fail(f"Function {function.name!r} uses '*' more than once.") + self.check_remaining_star() + self.keyword_only = True + else: + if self.keyword_only: + fail(f"Function {function.name!r}: '* [from ...]' must come before '*'") + if self.deprecated_positional: + fail(f"Function {function.name!r} uses '* [from ...]' more than once.") + self.deprecated_positional = version def parse_opening_square_bracket(self, function: Function) -> None: """Parse opening parameter group symbol '['.""" @@ -5460,11 +5565,38 @@ def parse_closing_square_bracket(self, function: Function) -> None: f"has an unsupported group configuration. " f"(Unexpected state {st}.c)") - def parse_slash(self, function: Function) -> None: - """Parse positional-only parameter marker '/'.""" - if self.positional_only: - fail(f"Function {function.name!r} uses '/' more than once.") + def parse_slash(self, function: Function, version: VersionTuple | None) -> None: + """Parse positional-only parameter marker '/'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.deprecated_keyword: + fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/' must precede '*'") + if self.positional_only: + fail(f"Function {function.name!r} uses '/' more than once.") + else: + if self.deprecated_keyword: + fail(f"Function {function.name!r} uses '/ [from ...]' more than once.") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") self.positional_only = True + self.deprecated_keyword = version + if version is not None: + found = False + for p in reversed(function.parameters.values()): + found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + break + if not found: + fail(f"Function {function.name!r} specifies '/ [from ...]' " + f"without preceding parameters.") # REQUIRED and OPTIONAL are allowed here, that allows positional-only # without option groups to work (and have default values!) allowed = { @@ -5476,19 +5608,13 @@ def parse_slash(self, function: Function) -> None: if (self.parameter_state not in allowed) or self.group: fail(f"Function {function.name!r} has an unsupported group configuration. " f"(Unexpected state {self.parameter_state}.d)") - if self.keyword_only: - fail(f"Function {function.name!r} mixes keyword-only and " - "positional-only parameters, which is unsupported.") # fixup preceding parameters for p in function.parameters.values(): - if p.is_vararg(): - continue - if (p.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD and - not isinstance(p.converter, self_converter) - ): - fail(f"Function {function.name!r} mixes keyword-only and " - "positional-only parameters, which is unsupported.") - p.kind = inspect.Parameter.POSITIONAL_ONLY + if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: + if version is None: + p.kind = inspect.Parameter.POSITIONAL_ONLY + else: + p.deprecated_keyword = version def state_parameter_docstring_start(self, line: str) -> None: assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" @@ -5773,6 +5899,29 @@ def format_docstring(self) -> str: signature=signature, parameters=parameters).rstrip() + def check_remaining_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + if self.keyword_only: + symbol = '*' + elif self.deprecated_positional: + symbol = '* [from ...]' + else: + return + + no_param_after_symbol = True + for p in reversed(self.function.parameters.values()): + if self.keyword_only: + if p.kind == inspect.Parameter.KEYWORD_ONLY: + return + elif self.deprecated_positional: + if p.deprecated_positional == self.deprecated_positional: + return + break + + fail(f"Function {self.function.name!r} specifies {symbol!r} " + f"without following parameters.", line_number=lineno) + def do_post_block_processing_cleanup(self, lineno: int) -> None: """ Called when processing the block is done. @@ -5780,28 +5929,7 @@ def do_post_block_processing_cleanup(self, lineno: int) -> None: if not self.function: return - def check_remaining( - symbol: str, - condition: Callable[[Parameter], bool] - ) -> None: - assert isinstance(self.function, Function) - - if values := self.function.parameters.values(): - last_param = next(reversed(values)) - no_param_after_symbol = condition(last_param) - else: - no_param_after_symbol = True - if no_param_after_symbol: - fname = self.function.full_name - fail(f"Function {fname!r} specifies {symbol!r} " - "without any parameters afterwards.", line_number=lineno) - - if self.keyword_only: - check_remaining("*", lambda p: p.kind != inspect.Parameter.KEYWORD_ONLY) - - if self.deprecated_positional: - check_remaining("* [from ...]", lambda p: not p.deprecated_positional) - + self.check_remaining_star(lineno) self.function.docstring = self.format_docstring() From ca0c6c1f1ef79d10bc49b61d638d87cde265aa94 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 19 Aug 2023 10:34:34 +0200 Subject: [PATCH 196/598] Docs: format sys.float_info properly (#108107) - Normalise capitalisation and punctuation - Use attribute markup for named tuple attributes - Use :c:macro: markup for C macros - Use a list for the 'rounds' attribute values - Use list-table, for better .rst readability - Remove one unneeded sys.float_info.dig link Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/sys.rst | 125 ++++++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 52 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index da2435aa4bc23d..a82e043f6209a2 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -574,61 +574,82 @@ always available. programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of floating types', for details. - .. tabularcolumns:: |l|l|L| - - +---------------------+---------------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+=====================+==================================================+ - | ``epsilon`` | ``DBL_EPSILON`` | difference between 1.0 and the least value | - | | | greater than 1.0 that is representable as a float| - | | | | - | | | See also :func:`math.ulp`. | - +---------------------+---------------------+--------------------------------------------------+ - | ``dig`` | ``DBL_DIG`` | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+---------------------+--------------------------------------------------+ - | ``mant_dig`` | ``DBL_MANT_DIG`` | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max`` | ``DBL_MAX`` | maximum representable positive finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_exp`` | ``DBL_MAX_EXP`` | maximum integer *e* such that ``radix**(e-1)`` is| - | | | a representable finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_10_exp`` | ``DBL_MAX_10_EXP`` | maximum integer *e* such that ``10**e`` is in the| - | | | range of representable finite floats | - +---------------------+---------------------+--------------------------------------------------+ - | ``min`` | ``DBL_MIN`` | minimum representable positive *normalized* float| - | | | | - | | | Use :func:`math.ulp(0.0) ` to get the | - | | | smallest positive *denormalized* representable | - | | | float. | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_exp`` | ``DBL_MIN_EXP`` | minimum integer *e* such that ``radix**(e-1)`` is| - | | | a normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_10_exp`` | ``DBL_MIN_10_EXP`` | minimum integer *e* such that ``10**e`` is a | - | | | normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``radix`` | ``FLT_RADIX`` | radix of exponent representation | - +---------------------+---------------------+--------------------------------------------------+ - | ``rounds`` | ``FLT_ROUNDS`` | integer representing the rounding mode for | - | | | floating-point arithmetic. This reflects the | - | | | value of the system ``FLT_ROUNDS`` macro at | - | | | interpreter startup time: | - | | | ``-1`` indeterminable, | - | | | ``0`` toward zero, | - | | | ``1`` to nearest, | - | | | ``2`` toward positive infinity, | - | | | ``3`` toward negative infinity | - | | | | - | | | All other values for ``FLT_ROUNDS`` characterize | - | | | implementation-defined rounding behavior. | - +---------------------+---------------------+--------------------------------------------------+ + .. list-table:: Attributes of the :data:`!float_info` :term:`named tuple` + :header-rows: 1 + + * - attribute + - float.h macro + - explanation + + * - .. attribute:: float_info.epsilon + - :c:macro:`!DBL_EPSILON` + - difference between 1.0 and the least value greater than 1.0 that is + representable as a float. + + See also :func:`math.ulp`. + + * - .. attribute:: float_info.dig + - :c:macro:`!DBL_DIG` + - The maximum number of decimal digits that can be faithfully + represented in a float; see below. + + * - .. attribute:: float_info.mant_dig + - :c:macro:`!DBL_MANT_DIG` + - Float precision: the number of base-``radix`` digits in the + significand of a float. + + * - .. attribute:: float_info.max + - :c:macro:`!DBL_MAX` + - The maximum representable positive finite float. + + * - .. attribute:: float_info.max_exp + - :c:macro:`!DBL_MAX_EXP` + - The maximum integer *e* such that ``radix**(e-1)`` is a representable + finite float. + + * - .. attribute:: float_info.max_10_exp + - :c:macro:`!DBL_MAX_10_EXP` + - The maximum integer *e* such that ``10**e`` is in the range of + representable finite floats. + + * - .. attribute:: float_info.min + - :c:macro:`!DBL_MIN` + - The minimum representable positive *normalized* float. + + Use :func:`math.ulp(0.0) ` to get the smallest positive + *denormalized* representable float. + + * - .. attribute:: float_info.min_exp + - :c:macro:`!DBL_MIN_EXP` + - The minimum integer *e* such that ``radix**(e-1)`` is a normalized + float. + + * - .. attribute:: float_info.min_10_exp + - :c:macro:`!DBL_MIN_10_EXP` + - The minimum integer *e* such that ``10**e`` is a normalized float. + + * - .. attribute:: float_info.radix + - :c:macro:`!FLT_RADIX` + - The radix of exponent representation. + + * - .. attribute:: float_info.rounds + - :c:macro:`!FLT_ROUNDS` + - An integer representing the rounding mode for floating-point arithmetic. + This reflects the value of the system :c:macro:`!FLT_ROUNDS` macro + at interpreter startup time: + + * ``-1``: indeterminable + * ``0``: toward zero + * ``1``: to nearest + * ``2``: toward positive infinity + * ``3``: toward negative infinity + + All other values for :c:macro:`!FLT_ROUNDS` characterize + implementation-defined rounding behavior. The attribute :attr:`sys.float_info.dig` needs further explanation. If ``s`` is any string representing a decimal number with at most - :attr:`sys.float_info.dig` significant digits, then converting ``s`` to a + :attr:`!sys.float_info.dig` significant digits, then converting ``s`` to a float and back again will recover a string representing the same decimal value:: From 79db9d9a0e8f51ad4ea5caae31d7ae4b29985271 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 19 Aug 2023 14:48:02 +0300 Subject: [PATCH 197/598] gh-72684: Tkinter: provide interface for "tk busy" subcommands (GH-107684) Add tkinter.Misc methods: tk_busy_hold(), tk_busy_configure(), tk_busy_cget(), tk_busy_forget(), tk_busy_current(), and tk_busy_status(). --- Doc/whatsnew/3.13.rst | 9 +++ Lib/test/test_tkinter/test_misc.py | 56 ++++++++++++- Lib/tkinter/__init__.py | 79 +++++++++++++++++++ ...3-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst | 3 + 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 85aa81b4a98d2c..47b868bad31923 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -144,6 +144,15 @@ pathlib :meth:`~pathlib.Path.is_dir`. (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) +tkinter +------- + +* Add :mod:`tkinter` widget methods: + :meth:`!tk_busy_hold`, :meth:`!tk_busy_configure`, + :meth:`!tk_busy_cget`, :meth:`!tk_busy_forget`, + :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. + (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) + traceback --------- diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index d1aca58d15fbd8..0ae27610be6078 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -1,9 +1,10 @@ import functools import unittest import tkinter +from tkinter import TclError import enum from test import support -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest +from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -36,6 +37,59 @@ def test_generated_names(self): for name in str(b).split('.'): self.assertFalse(name.isidentifier(), msg=repr(name)) + @requires_tk(8, 6, 6) + def test_tk_busy(self): + root = self.root + f = tkinter.Frame(root, name='myframe') + f2 = tkinter.Frame(root) + f.pack() + f2.pack() + b = tkinter.Button(f) + b.pack() + f.tk_busy_hold() + with self.assertRaisesRegex(TclError, 'unknown option "-spam"'): + f.tk_busy_configure(spam='eggs') + with self.assertRaisesRegex(TclError, 'unknown option "-spam"'): + f.tk_busy_cget('spam') + with self.assertRaisesRegex(TclError, 'unknown option "-spam"'): + f.tk_busy_configure('spam') + self.assertIsInstance(f.tk_busy_configure(), dict) + + self.assertTrue(f.tk_busy_status()) + self.assertFalse(root.tk_busy_status()) + self.assertFalse(f2.tk_busy_status()) + self.assertFalse(b.tk_busy_status()) + self.assertIn(f, f.tk_busy_current()) + self.assertIn(f, f.tk_busy_current('*.m?f*me')) + self.assertNotIn(f, f.tk_busy_current('*spam')) + + f.tk_busy_forget() + self.assertFalse(f.tk_busy_status()) + self.assertFalse(f.tk_busy_current()) + with self.assertRaisesRegex(TclError, "can't find busy window"): + f.tk_busy_configure() + with self.assertRaisesRegex(TclError, "can't find busy window"): + f.tk_busy_forget() + + @requires_tk(8, 6, 6) + def test_tk_busy_with_cursor(self): + root = self.root + if root._windowingsystem == 'aqua': + self.skipTest('the cursor option is not supported on OSX/Aqua') + f = tkinter.Frame(root, name='myframe') + f.pack() + f.tk_busy_hold(cursor='gumby') + + self.assertEqual(f.tk_busy_cget('cursor'), 'gumby') + f.tk_busy_configure(cursor='heart') + self.assertEqual(f.tk_busy_cget('cursor'), 'heart') + self.assertEqual(f.tk_busy_configure()['cursor'][4], 'heart') + self.assertEqual(f.tk_busy_configure('cursor')[4], 'heart') + + f.tk_busy_forget() + with self.assertRaisesRegex(TclError, "can't find busy window"): + f.tk_busy_cget('cursor') + def test_tk_setPalette(self): root = self.root root.tk_setPalette('black') diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c59f8d11e8a9da..440e7f100c8c47 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -901,6 +901,85 @@ def bell(self, displayof=0): """Ring a display's bell.""" self.tk.call(('bell',) + self._displayof(displayof)) + def tk_busy_cget(self, option): + """Return the value of busy configuration option. + + The widget must have been previously made busy by + tk_busy_hold(). Option may have any of the values accepted by + tk_busy_hold(). + """ + return self.tk.call('tk', 'busy', 'cget', self._w, '-'+option) + busy_cget = tk_busy_cget + + def tk_busy_configure(self, cnf=None, **kw): + """Query or modify the busy configuration options. + + The widget must have been previously made busy by + tk_busy_hold(). Options may have any of the values accepted by + tk_busy_hold(). + + Please note that the option database is referenced by the widget + name or class. For example, if a Frame widget with name "frame" + is to be made busy, the busy cursor can be specified for it by + either call: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + """ + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + return self._getconfigure( + 'tk', 'busy', 'configure', self._w) + if isinstance(cnf, str): + return self._getconfigure1( + 'tk', 'busy', 'configure', self._w, '-'+cnf) + self.tk.call('tk', 'busy', 'configure', self._w, *self._options(cnf)) + busy_config = busy_configure = tk_busy_config = tk_busy_configure + + def tk_busy_current(self, pattern=None): + """Return a list of widgets that are currently busy. + + If a pattern is given, only busy widgets whose path names match + a pattern are returned. + """ + return [self._nametowidget(x) for x in + self.tk.splitlist(self.tk.call( + 'tk', 'busy', 'current', pattern))] + busy_current = tk_busy_current + + def tk_busy_forget(self): + """Make this widget no longer busy. + + User events will again be received by the widget. + """ + self.tk.call('tk', 'busy', 'forget', self._w) + busy_forget = tk_busy_forget + + def tk_busy_hold(self, **kw): + """Make this widget appear busy. + + The specified widget and its descendants will be blocked from + user interactions. Normally update() should be called + immediately afterward to insure that the hold operation is in + effect before the application starts its processing. + + The only supported configuration option is: + + cursor: the cursor to be displayed when the widget is made + busy. + """ + self.tk.call('tk', 'busy', 'hold', self._w, *self._options(kw)) + busy = busy_hold = tk_busy = tk_busy_hold + + def tk_busy_status(self): + """Return True if the widget is busy, False otherwise.""" + return self.tk.getboolean(self.tk.call( + 'tk', 'busy', 'status', self._w)) + busy_status = tk_busy_status + # Clipboard handling: def clipboard_get(self, **kw): """Retrieve data from the clipboard on window's display. diff --git a/Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst b/Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst new file mode 100644 index 00000000000000..1ac0e6d790456c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst @@ -0,0 +1,3 @@ +Add :mod:`tkinter` widget methods: :meth:`!tk_busy_hold`, +:meth:`!tk_busy_configure`, :meth:`!tk_busy_cget`, :meth:`!tk_busy_forget`, +:meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. From 633ea217a85f6b6ba5bdbc73094254d5811b3485 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 19 Aug 2023 14:51:03 +0300 Subject: [PATCH 198/598] gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) Such C API functions as PyErr_SetString(), PyErr_Format(), PyErr_SetFromErrnoWithFilename() and many others no longer crash or ignore errors if it failed to format the error message or decode the filename. Instead, they keep a corresponding error. --- Lib/test/test_capi/test_exceptions.py | 81 +++++++++++++++++++ ...-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst | 4 + Modules/_testcapi/clinic/exceptions.c.h | 64 ++++++++++++++- Modules/_testcapi/exceptions.c | 41 ++++++++++ Python/errors.c | 37 +++++++-- 5 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 118b575cba6df7..b96cc7922a0ee7 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -1,9 +1,12 @@ +import errno +import os import re import sys import unittest from test import support from test.support import import_helper +from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE from test.support.script_helper import assert_python_failure from test.support.testcase import ExceptionIsLikeMixin @@ -12,6 +15,8 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +NULL = None + class Test_Exceptions(unittest.TestCase): def test_exception(self): @@ -189,6 +194,82 @@ def __repr__(self): self.assertEqual(exc.__notes__[0], 'Normalization failed: type=Broken args=') + def test_set_string(self): + """Test PyErr_SetString()""" + setstring = _testcapi.err_setstring + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, b'error') + self.assertEqual(e.exception.args, ('error',)) + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, 'помилка'.encode()) + self.assertEqual(e.exception.args, ('помилка',)) + + with self.assertRaises(UnicodeDecodeError): + setstring(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, setstring, list, b'error') + # CRASHES setstring(ZeroDivisionError, NULL) + # CRASHES setstring(NULL, b'error') + + def test_format(self): + """Test PyErr_Format()""" + import_helper.import_module('ctypes') + from ctypes import pythonapi, py_object, c_char_p, c_int + name = "PyErr_Format" + PyErr_Format = getattr(pythonapi, name) + PyErr_Format.argtypes = (py_object, c_char_p,) + PyErr_Format.restype = py_object + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s %d', b'error', c_int(42)) + self.assertEqual(e.exception.args, ('error 42',)) + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s', 'помилка'.encode()) + self.assertEqual(e.exception.args, ('помилка',)) + + with self.assertRaisesRegex(OverflowError, 'not in range'): + PyErr_Format(ZeroDivisionError, b'%c', c_int(-1)) + with self.assertRaisesRegex(ValueError, 'format string'): + PyErr_Format(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, PyErr_Format, list, b'error') + # CRASHES PyErr_Format(ZeroDivisionError, NULL) + # CRASHES PyErr_Format(py_object(), b'error') + + def test_setfromerrnowithfilename(self): + """Test PyErr_SetFromErrnoWithFilename()""" + setfromerrnowithfilename = _testcapi.err_setfromerrnowithfilename + ENOENT = errno.ENOENT + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory')) + self.assertEqual(e.exception.errno, ENOENT) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, os.fsencode(TESTFN)) + self.assertEqual(e.exception.filename, TESTFN) + + if TESTFN_UNDECODABLE: + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, TESTFN_UNDECODABLE) + self.assertEqual(e.exception.filename, + os.fsdecode(TESTFN_UNDECODABLE)) + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, NULL) + self.assertIsNone(e.exception.filename) + + with self.assertRaises(OSError) as e: + setfromerrnowithfilename(0, OSError, b'file') + self.assertEqual(e.exception.args, (0, 'Error')) + self.assertEqual(e.exception.errno, 0) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(ZeroDivisionError) as e: + setfromerrnowithfilename(ENOENT, ZeroDivisionError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory', 'file')) + # CRASHES setfromerrnowithfilename(ENOENT, NULL, b'error') + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): diff --git a/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst new file mode 100644 index 00000000000000..58ee3f169a28cc --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst @@ -0,0 +1,4 @@ +Such C API functions as ``PyErr_SetString()``, ``PyErr_Format()``, +``PyErr_SetFromErrnoWithFilename()`` and many others no longer crash or +ignore errors if it failed to format the error message or decode the +filename. Instead, they keep a corresponding error. diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 01730ffa2ed036..16954a5ebf3e58 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -215,6 +215,68 @@ _testcapi_exc_set_object_fetch(PyObject *module, PyObject *const *args, Py_ssize return return_value; } +PyDoc_STRVAR(_testcapi_err_setstring__doc__, +"err_setstring($module, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETSTRING_METHODDEF \ + {"err_setstring", _PyCFunction_CAST(_testcapi_err_setstring), METH_FASTCALL, _testcapi_err_setstring__doc__}, + +static PyObject * +_testcapi_err_setstring_impl(PyObject *module, PyObject *exc, + const char *value, Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setstring(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "Oz#:err_setstring", + &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setstring_impl(module, exc, value, value_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testcapi_err_setfromerrnowithfilename__doc__, +"err_setfromerrnowithfilename($module, error, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF \ + {"err_setfromerrnowithfilename", _PyCFunction_CAST(_testcapi_err_setfromerrnowithfilename), METH_FASTCALL, _testcapi_err_setfromerrnowithfilename__doc__}, + +static PyObject * +_testcapi_err_setfromerrnowithfilename_impl(PyObject *module, int error, + PyObject *exc, const char *value, + Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setfromerrnowithfilename(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int error; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "iOz#:err_setfromerrnowithfilename", + &error, &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setfromerrnowithfilename_impl(module, error, exc, value, value_length); + +exit: + return return_value; +} + PyDoc_STRVAR(_testcapi_raise_exception__doc__, "raise_exception($module, exception, num_args, /)\n" "--\n" @@ -426,4 +488,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=fd6aef54f195c77b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d574342d716e98b5 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index a627bf1717fe0c..d7edf8c7ce6744 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -1,6 +1,8 @@ #include "parts.h" #include "clinic/exceptions.c.h" +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + /*[clinic input] module _testcapi [clinic start generated code]*/ @@ -129,6 +131,43 @@ _testcapi_exc_set_object_fetch_impl(PyObject *module, PyObject *exc, return value; } +/*[clinic input] +_testcapi.err_setstring + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setstring_impl(PyObject *module, PyObject *exc, + const char *value, Py_ssize_t value_length) +/*[clinic end generated code: output=fba8705e5703dd3f input=e8a95fad66d9004b]*/ +{ + NULLABLE(exc); + PyErr_SetString(exc, value); + return NULL; +} + +/*[clinic input] +_testcapi.err_setfromerrnowithfilename + error: int + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setfromerrnowithfilename_impl(PyObject *module, int error, + PyObject *exc, const char *value, + Py_ssize_t value_length) +/*[clinic end generated code: output=d02df5749a01850e input=ff7c384234bf097f]*/ +{ + NULLABLE(exc); + errno = error; + PyErr_SetFromErrnoWithFilename(exc, value); + return NULL; +} + /*[clinic input] _testcapi.raise_exception exception as exc: object @@ -338,6 +377,8 @@ static PyMethodDef test_methods[] = { _TESTCAPI_MAKE_EXCEPTION_WITH_DOC_METHODDEF _TESTCAPI_EXC_SET_OBJECT_METHODDEF _TESTCAPI_EXC_SET_OBJECT_FETCH_METHODDEF + _TESTCAPI_ERR_SETSTRING_METHODDEF + _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF _TESTCAPI_RAISE_EXCEPTION_METHODDEF _TESTCAPI_RAISE_MEMORYERROR_METHODDEF _TESTCAPI_SET_EXC_INFO_METHODDEF diff --git a/Python/errors.c b/Python/errors.c index 916958c9a0ab00..d9086c507251e6 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -292,8 +292,10 @@ _PyErr_SetString(PyThreadState *tstate, PyObject *exception, const char *string) { PyObject *value = PyUnicode_FromString(string); - _PyErr_SetObject(tstate, exception, value); - Py_XDECREF(value); + if (value != NULL) { + _PyErr_SetObject(tstate, exception, value); + Py_DECREF(value); + } } void @@ -891,7 +893,13 @@ PyErr_SetFromErrnoWithFilenameObjects(PyObject *exc, PyObject *filenameObject, P PyObject * PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); return result; @@ -988,7 +996,13 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *ret = PyErr_SetExcFromWindowsErrWithFilenameObjects(exc, ierr, name, @@ -1012,7 +1026,13 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetExcFromWindowsErrWithFilenameObjects( PyExc_OSError, ierr, name, NULL); @@ -1137,9 +1157,10 @@ _PyErr_FormatV(PyThreadState *tstate, PyObject *exception, _PyErr_Clear(tstate); string = PyUnicode_FromFormatV(format, vargs); - - _PyErr_SetObject(tstate, exception, string); - Py_XDECREF(string); + if (string != NULL) { + _PyErr_SetObject(tstate, exception, string); + Py_DECREF(string); + } return NULL; } From c31c61c04e55ef431615ffec959d84ac73a3db81 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 19 Aug 2023 19:03:26 +0200 Subject: [PATCH 199/598] Docs: Remove links to external C functions and macros in os.rst (#108138) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/os.rst | 52 ++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7d63ac15027d51..4f4d8c99023529 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -215,7 +215,7 @@ process and user. On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for - :c:func:`putenv`. + :c:func:`!putenv`. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from @@ -564,7 +564,7 @@ process and user. .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + cause memory leaks. Refer to the system documentation for :c:func:`!putenv`. .. audit-event:: os.putenv key,value os.putenv @@ -646,7 +646,7 @@ process and user. .. function:: setpgrp() - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on + Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -654,7 +654,7 @@ process and user. .. function:: setpgid(pid, pgrp, /) - Call the system call :c:func:`setpgid` to set the process group id of the + Call the system call :c:func:`!setpgid` to set the process group id of the process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. @@ -1077,7 +1077,7 @@ as internal buffering of data. .. function:: fsync(fd) Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`fsync` function; on Windows, the MS :c:func:`_commit` function. + native :c:func:`!fsync` function; on Windows, the MS :c:func:`!_commit` function. If you're starting with a buffered Python :term:`file object` *f*, first do ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal @@ -2304,7 +2304,7 @@ features: .. function:: lstat(path, *, dir_fd=None) - Perform the equivalent of an :c:func:`lstat` system call on the given path. + Perform the equivalent of an :c:func:`!lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a :class:`stat_result` object. @@ -3147,14 +3147,16 @@ features: Windows file attributes: ``dwFileAttributes`` member of the ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` + :c:func:`!GetFileInformationByHandle`. + See the :const:`!FILE_ATTRIBUTE_* ` constants in the :mod:`stat` module. .. attribute:: st_reparse_tag - When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` set, this field contains the tag identifying the type of reparse point. - See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + See the :const:`IO_REPARSE_TAG_* ` + constants in the :mod:`stat` module. The standard module :mod:`stat` defines functions and constants that are useful for extracting information from a :c:struct:`stat` structure. (On @@ -3212,7 +3214,7 @@ features: .. function:: statvfs(path) - Perform a :c:func:`statvfs` system call on the given path. The return value is + Perform a :c:func:`!statvfs` system call on the given path. The return value is an object whose attributes describe the filesystem on the given path, and correspond to the members of the :c:struct:`statvfs` structure, namely: :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, @@ -4303,7 +4305,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawn` C library API for use from Python. + Wraps the :c:func:`!posix_spawn` C library API for use from Python. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. @@ -4339,16 +4341,16 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. These tuples correspond to the C library - :c:func:`posix_spawn_file_actions_addopen`, - :c:func:`posix_spawn_file_actions_addclose`, and - :c:func:`posix_spawn_file_actions_adddup2` API calls used to prepare - for the :c:func:`posix_spawn` call itself. + :c:func:`!posix_spawn_file_actions_addopen`, + :c:func:`!posix_spawn_file_actions_addclose`, and + :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value specified. If the value specified is 0, the child's process group ID will be made the same as its process ID. If the value of *setpgroup* is not set, the child will inherit the parent's process group ID. This argument corresponds - to the C library :c:macro:`POSIX_SPAWN_SETPGROUP` flag. + to the C library :c:macro:`!POSIX_SPAWN_SETPGROUP` flag. If the *resetids* argument is ``True`` it will reset the effective UID and GID of the child to the real UID and GID of the parent process. If the @@ -4356,27 +4358,27 @@ written in Python, such as a mail server's external command delivery program. the parent. In either case, if the set-user-ID and set-group-ID permission bits are enabled on the executable file, their effect will override the setting of the effective UID and GID. This argument corresponds to the C - library :c:macro:`POSIX_SPAWN_RESETIDS` flag. + library :c:macro:`!POSIX_SPAWN_RESETIDS` flag. If the *setsid* argument is ``True``, it will create a new session ID - for ``posix_spawn``. *setsid* requires :c:macro:`POSIX_SPAWN_SETSID` - or :c:macro:`POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` + for ``posix_spawn``. *setsid* requires :c:macro:`!POSIX_SPAWN_SETSID` + or :c:macro:`!POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` is raised. The *setsigmask* argument will set the signal mask to the signal set specified. If the parameter is not used, then the child inherits the parent's signal mask. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGMASK` flag. + :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. The *sigdef* argument will reset the disposition of all signals in the set specified. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGDEF` flag. + :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. The *scheduler* argument must be a tuple containing the (optional) scheduler policy and an instance of :class:`sched_param` with the scheduler parameters. A value of ``None`` in the place of the scheduler policy indicates that is not being provided. This argument is a combination of the C library - :c:macro:`POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`POSIX_SPAWN_SETSCHEDULER` + :c:macro:`!POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`!POSIX_SPAWN_SETSCHEDULER` flags. .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn @@ -4389,7 +4391,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawnp` C library API for use from Python. + Wraps the :c:func:`!posix_spawnp` C library API for use from Python. Similar to :func:`posix_spawn` except that the system searches for the *executable* file in the list of directories specified by the @@ -4570,7 +4572,7 @@ written in Python, such as a mail server's external command delivery program. Use *show_cmd* to override the default window style. Whether this has any effect will depend on the application being launched. Values are integers as - supported by the Win32 :c:func:`ShellExecute` function. + supported by the Win32 :c:func:`!ShellExecute` function. :func:`startfile` returns as soon as the associated application is launched. There is no option to wait for the application to close, and no way to retrieve @@ -4580,7 +4582,7 @@ written in Python, such as a mail server's external command delivery program. :func:`os.path.normpath` function to ensure that paths are properly encoded for Win32. - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` + To reduce interpreter startup overhead, the Win32 :c:func:`!ShellExecute` function is not resolved until this function is first called. If the function cannot be resolved, :exc:`NotImplementedError` will be raised. From a47c13cae5b32e6f3d7532cc6dbb4e1ac31219de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 19 Aug 2023 22:14:38 +0200 Subject: [PATCH 200/598] gh-107980: fix doc role for asyncio.timeouts (#108126) --- Doc/library/asyncio-task.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 0651ff7213e527..1c419728c9a482 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -631,9 +631,9 @@ Shielding From Cancellation Timeouts ======== -.. coroutinefunction:: timeout(delay) +.. function:: timeout(delay) - An :ref:`asynchronous context manager ` + Return an :ref:`asynchronous context manager ` that can be used to limit the amount of time spent waiting on something. @@ -724,7 +724,7 @@ Timeouts .. versionadded:: 3.11 -.. coroutinefunction:: timeout_at(when) +.. function:: timeout_at(when) Similar to :func:`asyncio.timeout`, except *when* is the absolute time to stop waiting, or ``None``. From 1dc0c58d2b17819720d184ec0287a8a9b1dc347e Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Sun, 20 Aug 2023 05:11:39 -0300 Subject: [PATCH 201/598] Fix misspellings in sysconfig docs (#108156) --- Doc/library/sysconfig.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index c805c50ffc689f..26344ea4e7e2ab 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -84,13 +84,13 @@ Python currently supports nine schemes: through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. - *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX - platforms; by default it is the same as *posix_prefix* . + platforms; by default it is the same as *posix_prefix*. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. - *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt* . -- *venv*: a scheme with values from ether *posix_venv* or *nt_venv* depending - on the platform Python runs on + platforms; by default it is the same as *nt*. +- *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending + on the platform Python runs on. - *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique @@ -187,7 +187,7 @@ identifier. Python currently uses eight paths: platform is used. If *vars* is provided, it must be a dictionary of variables that will update - the dictionary return by :func:`get_config_vars`. + the dictionary returned by :func:`get_config_vars`. If *expand* is set to ``False``, the path will not be expanded using the variables. From b1e5d2c601bbd3d435b60deef4818f3622bdfca3 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 20 Aug 2023 14:30:42 +0530 Subject: [PATCH 202/598] Fix patchcheck for `asyncio.tasks` (#108159) --- Lib/asyncio/tasks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 75dd3cb6c2e987..edc64fda2a6ad6 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -935,21 +935,21 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in + Most applications will use `Task` for `custom_task_constructor` and in this case there's no need to call `create_eager_task_factory()` directly. Instead the global `eager_task_factory` instance can be used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( From 1a713eac47b26899044752f02cbfcb4d628dda2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hadh=C3=A1zy=20Tam=C3=A1s?= <85063808+Hels15@users.noreply.github.com> Date: Sun, 20 Aug 2023 11:33:15 +0100 Subject: [PATCH 203/598] gh-107619: Extend functools LRU cache docs with generators and async functions (#107934) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Kumar Aditya --- Doc/library/functools.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 40f43f8b3519cd..f736eb0aca1ac5 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -226,8 +226,9 @@ The :mod:`functools` module defines the following functions: In general, the LRU cache should only be used when you want to reuse previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create distinct mutable - objects on each call, or impure functions such as time() or random(). + functions with side-effects, functions that need to create + distinct mutable objects on each call (such as generators and async functions), + or impure functions such as time() or random(). Example of an LRU cache for static web content:: From beffb30dc7a07044f4198245d049ddda1f4b24db Mon Sep 17 00:00:00 2001 From: Tomas R Date: Sun, 20 Aug 2023 12:54:06 +0200 Subject: [PATCH 204/598] gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (#107769) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Kumar Aditya --- Doc/library/ctypes.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 474359a5cd98bd..8af609d53ff4b8 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2029,17 +2029,17 @@ Utility functions specifying an address, or a ctypes instance. -.. function:: POINTER(type) +.. function:: POINTER(type, /) - This factory function creates and returns a new ctypes pointer type. Pointer - types are cached and reused internally, so calling this function repeatedly is - cheap. *type* must be a ctypes type. + Create and return a new ctypes pointer type. Pointer types are cached and + reused internally, so calling this function repeatedly is cheap. + *type* must be a ctypes type. -.. function:: pointer(obj) +.. function:: pointer(obj, /) - This function creates a new pointer instance, pointing to *obj*. The returned - object is of the type ``POINTER(type(obj))``. + Create a new pointer instance, pointing to *obj*. + The returned object is of the type ``POINTER(type(obj))``. Note: If you just want to pass a pointer to an object to a foreign function call, you should use ``byref(obj)`` which is much faster. From 29fa7afef94d74e18d97485c085d1ccf80c16ca3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 20 Aug 2023 13:09:33 +0200 Subject: [PATCH 205/598] Docs: Fix Sphinx warnings in sys.rst (#108106) - Mark up named tuple attributes as attributes - Remove links for external functions - io.BufferedIOBase has no 'buffer' attribute; remove the link and mark up using :attr:`!buffer` - (Re)format some tables as bullet lists: - sys._emscripten_info - sys.hash_info - sys.int_info - sys.thread_info - In the paragraphs mentioning 'f_trace_lines' and 'f_trace_opcodes', add links to the frame objects reference. Co-authored-by: Alex Waygood Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/sys.rst | 265 ++++++++++++++++++++++----------------- Doc/library/textwrap.rst | 2 +- Doc/tools/.nitignore | 1 - 3 files changed, 153 insertions(+), 115 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a82e043f6209a2..e99ab90bd18e4d 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -333,23 +333,21 @@ always available. *wasm32-emscripten* platform. The named tuple is provisional and may change in the future. - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------------+ - | Attribute | Explanation | - +=============================+==============================================+ - | :const:`emscripten_version` | Emscripten version as tuple of ints | - | | (major, minor, micro), e.g. ``(3, 1, 8)``. | - +-----------------------------+----------------------------------------------+ - | :const:`runtime` | Runtime string, e.g. browser user agent, | - | | ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. | - +-----------------------------+----------------------------------------------+ - | :const:`pthreads` | ``True`` if Python is compiled with | - | | Emscripten pthreads support. | - +-----------------------------+----------------------------------------------+ - | :const:`shared_memory` | ``True`` if Python is compiled with shared | - | | memory support. | - +-----------------------------+----------------------------------------------+ + .. attribute:: _emscripten_info.emscripten_version + + Emscripten version as tuple of ints (major, minor, micro), e.g. ``(3, 1, 8)``. + + .. attribute:: _emscripten_info.runtime + + Runtime string, e.g. browser user agent, ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. + + .. attribute:: _emscripten_info.pthreads + + ``True`` if Python is compiled with Emscripten pthreads support. + + .. attribute:: _emscripten_info.shared_memory + + ``True`` if Python is compiled with shared memory support. .. availability:: Emscripten. @@ -515,28 +513,62 @@ always available. The :term:`named tuple` *flags* exposes the status of command line flags. The attributes are read only. - ============================== ============================================================================================================== - attribute flag - ============================== ============================================================================================================== - :const:`debug` :option:`-d` - :const:`inspect` :option:`-i` - :const:`interactive` :option:`-i` - :const:`isolated` :option:`-I` - :const:`optimize` :option:`-O` or :option:`-OO` - :const:`dont_write_bytecode` :option:`-B` - :const:`no_user_site` :option:`-s` - :const:`no_site` :option:`-S` - :const:`ignore_environment` :option:`-E` - :const:`verbose` :option:`-v` - :const:`bytes_warning` :option:`-b` - :const:`quiet` :option:`-q` - :const:`hash_randomization` :option:`-R` - :const:`dev_mode` :option:`-X dev <-X>` (:ref:`Python Development Mode `) - :const:`utf8_mode` :option:`-X utf8 <-X>` - :const:`safe_path` :option:`-P` - :const:`int_max_str_digits` :option:`-X int_max_str_digits <-X>` (:ref:`integer string conversion length limitation `) - :const:`warn_default_encoding` :option:`-X warn_default_encoding <-X>` - ============================== ============================================================================================================== + .. list-table:: + + * - .. attribute:: flags.debug + - :option:`-d` + + * - .. attribute:: flags.inspect + - :option:`-i` + + * - .. attribute:: flags.interactive + - :option:`-i` + + * - .. attribute:: flags.isolated + - :option:`-I` + + * - .. attribute:: flags.optimize + - :option:`-O` or :option:`-OO` + + * - .. attribute:: flags.dont_write_bytecode + - :option:`-B` + + * - .. attribute:: flags.no_user_site + - :option:`-s` + + * - .. attribute:: flags.no_site + - :option:`-S` + + * - .. attribute:: flags.ignore_environment + - :option:`-E` + + * - .. attribute:: flags.verbose + - :option:`-v` + + * - .. attribute:: flags.bytes_warning + - :option:`-b` + + * - .. attribute:: flags.quiet + - :option:`-q` + + * - .. attribute:: flags.hash_randomization + - :option:`-R` + + * - .. attribute:: flags.dev_mode + - :option:`-X dev <-X>` (:ref:`Python Development Mode `) + + * - .. attribute:: flags.utf8_mode + - :option:`-X utf8 <-X>` + + * - .. attribute:: flags.safe_path + - :option:`-P` + + * - .. attribute:: flags.int_max_str_digits + - :option:`-X int_max_str_digits <-X>` + (:ref:`integer string conversion length limitation `) + + * - .. attribute:: flags.warn_default_encoding + - :option:`-X warn_default_encoding <-X>` .. versionchanged:: 3.2 Added ``quiet`` attribute for the new :option:`-q` flag. @@ -923,8 +955,8 @@ always available. | | a domain controller. | +---------------------------------------+---------------------------------+ - This function wraps the Win32 :c:func:`GetVersionEx` function; see the - Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information + This function wraps the Win32 :c:func:`!GetVersionEx` function; see the + Microsoft documentation on :c:func:`!OSVERSIONINFOEX` for more information about these fields. *platform_version* returns the major version, minor version and @@ -982,28 +1014,37 @@ always available. implementation. For more details about hashing of numeric types, see :ref:`numeric-hash`. - +---------------------+--------------------------------------------------+ - | attribute | explanation | - +=====================+==================================================+ - | :const:`width` | width in bits used for hash values | - +---------------------+--------------------------------------------------+ - | :const:`modulus` | prime modulus P used for numeric hash scheme | - +---------------------+--------------------------------------------------+ - | :const:`inf` | hash value returned for a positive infinity | - +---------------------+--------------------------------------------------+ - | :const:`nan` | (this attribute is no longer used) | - +---------------------+--------------------------------------------------+ - | :const:`imag` | multiplier used for the imaginary part of a | - | | complex number | - +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ + .. attribute:: hash_info.width + + The width in bits used for hash values + + .. attribute:: hash_info.modulus + The prime modulus P used for numeric hash scheme + + .. attribute:: hash_info.inf + + The hash value returned for a positive infinity + + .. attribute:: hash_info.nan + + (This attribute is no longer used) + + .. attribute:: hash_info.imag + + The multiplier used for the imaginary part of a complex number + + .. attribute:: hash_info.algorithm + + The name of the algorithm for hashing of str, bytes, and memoryview + + .. attribute:: hash_info.hash_bits + + The internal output size of the hash algorithm + + .. attribute:: hash_info.seed_bits + + The size of the seed key of the hash algorithm .. versionadded:: 3.2 @@ -1081,32 +1122,31 @@ always available. A :term:`named tuple` that holds information about Python's internal representation of integers. The attributes are read only. - .. tabularcolumns:: |l|L| - - +----------------------------------------+-----------------------------------------------+ - | Attribute | Explanation | - +========================================+===============================================+ - | :const:`bits_per_digit` | number of bits held in each digit. Python | - | | integers are stored internally in base | - | | ``2**int_info.bits_per_digit`` | - +----------------------------------------+-----------------------------------------------+ - | :const:`sizeof_digit` | size in bytes of the C type used to | - | | represent a digit | - +----------------------------------------+-----------------------------------------------+ - | :const:`default_max_str_digits` | default value for | - | | :func:`sys.get_int_max_str_digits` when it | - | | is not otherwise explicitly configured. | - +----------------------------------------+-----------------------------------------------+ - | :const:`str_digits_check_threshold` | minimum non-zero value for | - | | :func:`sys.set_int_max_str_digits`, | - | | :envvar:`PYTHONINTMAXSTRDIGITS`, or | - | | :option:`-X int_max_str_digits <-X>`. | - +----------------------------------------+-----------------------------------------------+ + .. attribute:: int_info.bits_per_digit + + The number of bits held in each digit. + Python integers are stored internally in base ``2**int_info.bits_per_digit``. + + .. attribute:: int_info.sizeof_digit + + The size in bytes of the C type used to represent a digit. + + .. attribute:: int_info.default_max_str_digits + + The default value for :func:`sys.get_int_max_str_digits` + when it is not otherwise explicitly configured. + + .. attribute:: int_info.str_digits_check_threshold + + The minimum non-zero value for :func:`sys.set_int_max_str_digits`, + :envvar:`PYTHONINTMAXSTRDIGITS`, or :option:`-X int_max_str_digits <-X>`. .. versionadded:: 3.1 .. versionchanged:: 3.11 - Added ``default_max_str_digits`` and ``str_digits_check_threshold``. + + Added :attr:`~int_info.default_max_str_digits` and + :attr:`~int_info.str_digits_check_threshold`. .. data:: __interactivehook__ @@ -1533,7 +1573,7 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`f_trace_lines` to :const:`False` on that frame. + :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame `. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1551,8 +1591,8 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`f_trace_opcodes` to :const:`True` on the - frame. + requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the + :ref:`frame `. Note that as an exception is propagated down the chain of callers, an ``'exception'`` event is generated at each level. @@ -1581,8 +1621,8 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`f_trace_lines` and - :attr:`f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`!f_trace_lines` and + :attr:`!f_trace_opcodes` attributes added to frames .. function:: set_asyncgen_hooks(firstiter, finalizer) @@ -1739,7 +1779,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. + do not support the :attr:!buffer` attribute. .. data:: __stdin__ @@ -1787,29 +1827,28 @@ always available. A :term:`named tuple` holding information about the thread implementation. - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +------------------+---------------------------------------------------------+ - | Attribute | Explanation | - +==================+=========================================================+ - | :const:`name` | Name of the thread implementation: | - | | | - | | * ``'nt'``: Windows threads | - | | * ``'pthread'``: POSIX threads | - | | * ``'pthread-stubs'``: stub POSIX threads | - | | (on WebAssembly platforms without threading support) | - | | * ``'solaris'``: Solaris threads | - +------------------+---------------------------------------------------------+ - | :const:`lock` | Name of the lock implementation: | - | | | - | | * ``'semaphore'``: a lock uses a semaphore | - | | * ``'mutex+cond'``: a lock uses a mutex | - | | and a condition variable | - | | * ``None`` if this information is unknown | - +------------------+---------------------------------------------------------+ - | :const:`version` | Name and version of the thread library. It is a string, | - | | or ``None`` if this information is unknown. | - +------------------+---------------------------------------------------------+ + .. attribute:: thread_info.name + + The name of the thread implementation: + + * ``"nt"``: Windows threads + * ``"pthread"``: POSIX threads + * ``"pthread-stubs"``: stub POSIX threads + (on WebAssembly platforms without threading support) + * ``"solaris"``: Solaris threads + + .. attribute:: thread_info.lock + + The name of the lock implementation: + + * ``"semaphore"``: a lock uses a semaphore + * ``"mutex+cond"``: a lock uses a mutex and a condition variable + * ``None`` if this information is unknown + + .. attribute:: thread_info.version + + The name and version of the thread library. + It is a string, or ``None`` if this information is unknown. .. versionadded:: 3.3 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a150eefbf932ef..e2952ce3cc2ca3 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -60,7 +60,7 @@ functions should be good enough; otherwise, you should use an instance of First the whitespace in *text* is collapsed (all whitespace is replaced by single spaces). If the result fits in the *width*, it is returned. Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`.placeholder` fit within :attr:`.width`:: + plus the *placeholder* fit within *width*:: >>> textwrap.shorten("Hello world!", width=12) 'Hello world!' diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 654287cc50b55b..622237e14318ad 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -127,7 +127,6 @@ Doc/library/ssl.rst Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/sys.rst Doc/library/sys_path_init.rst Doc/library/syslog.rst Doc/library/tarfile.rst From 4d4393139fae39db26dead33529b6ae0bafbfc58 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 20 Aug 2023 13:10:48 +0200 Subject: [PATCH 206/598] Docs: Fix Sphinx warnings in license.rst (#108142) - Fix links to stdlib modules - Silence links to external functions --- Doc/license.rst | 10 +++++----- Doc/tools/.nitignore | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/license.rst b/Doc/license.rst index be8efa70611378..82039fd9aaf9d0 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -352,8 +352,8 @@ the verbatim comments from the original code:: Sockets ------- -The :mod:`socket` module uses the functions, :func:`getaddrinfo`, and -:func:`getnameinfo`, which are coded in separate source files from the WIDE +The :mod:`socket` module uses the functions, :c:func:`!getaddrinfo`, and +:c:func:`!getnameinfo`, which are coded in separate source files from the WIDE Project, https://www.wide.ad.jp/. :: Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -387,7 +387,7 @@ Project, https://www.wide.ad.jp/. :: Asynchronous socket services ---------------------------- -The :mod:`test.support.asynchat` and :mod:`test.support.asyncore` +The :mod:`!test.support.asynchat` and :mod:`!test.support.asyncore` modules contain the following notice:: Copyright 1996 by Sam Rushing @@ -539,7 +539,7 @@ The :mod:`xmlrpc.client` module contains the following notice:: test_epoll ---------- -The :mod:`test_epoll` module contains the following notice:: +The :mod:`!test.test_epoll` module contains the following notice:: Copyright (c) 2001-2006 Twisted Matrix Laboratories. @@ -844,7 +844,7 @@ and later releases derived from that, the Apache License v2 applies:: expat ----- -The :mod:`pyexpat` extension is built using an included copy of the expat +The :mod:`pyexpat ` extension is built using an included copy of the expat sources unless the build is configured ``--with-system-expat``:: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 622237e14318ad..ebadd327591971 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -160,7 +160,6 @@ Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst Doc/library/zlib.rst -Doc/license.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst From c735e79afb62324624864e1943f84825249f58ed Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 20 Aug 2023 13:19:07 +0200 Subject: [PATCH 207/598] Docs: Fix Sphinx warnings in logging.rst (GH-108139) Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/library/logging.rst | 50 ++++++++++++++++++++++------------------- Doc/tools/.nitignore | 1 - 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 49e870e9e2479b..97a0b82b78b1b3 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -439,7 +439,7 @@ Handler Objects Handlers have the following attributes and methods. Note that :class:`Handler` is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call +subclasses. However, the :meth:`!__init__` method in subclasses needs to call :meth:`Handler.__init__`. .. class:: Handler @@ -1019,30 +1019,34 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. + In addition to the above, :class:`LoggerAdapter` supports the following + methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, + :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, + :meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, + :meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and + :meth:`~Logger.hasHandlers`. These methods have the same signatures as their + counterparts in :class:`Logger`, so you can use the two types of instances + interchangeably. -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + .. versionchanged:: 3.2 + + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + .. versionchanged:: 3.6 -.. versionchanged:: 3.6 - Attribute :attr:`manager` and method :meth:`_log` were added, which - delegate to the underlying logger and allow adapters to be nested. + Attribute :attr:`!manager` and method :meth:`!_log` were added, which + delegate to the underlying logger and allow adapters to be nested. -.. versionchanged:: 3.13 - Remove the undocumented ``warn()`` method which was an alias to the - ``warning()`` method. + .. versionchanged:: 3.13 + + Remove the undocumented :meth:`!warn`` method which was an alias to the + :meth:`!warning` method. + + .. versionchanged:: 3.13 -.. versionchanged:: 3.13 - The *merge_extra* argument was added. + The *merge_extra* argument was added. Thread Safety @@ -1430,8 +1434,8 @@ functions. .. function:: setLoggerClass(klass) Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + The class should define :meth:`!__init__` such that only a name argument is + required, and the :meth:`!__init__` should call :meth:`!Logger.__init__`. This function is typically called before any loggers are instantiated by applications which need to use custom logger behavior. After this call, as at any other time, do not instantiate loggers directly using the subclass: continue to use diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ebadd327591971..f7e454af62c87f 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -92,7 +92,6 @@ Doc/library/inspect.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst -Doc/library/logging.rst Doc/library/lzma.rst Doc/library/mailbox.rst Doc/library/mmap.rst From a390ec20f5a85b9c16e8708f117667783d08863c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 20 Aug 2023 14:53:28 +0100 Subject: [PATCH 208/598] Resolve reference warnings in faq/programming.rst (#108150) --- Doc/faq/programming.rst | 6 +++--- Doc/tools/.nitignore | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 6e1812504a184f..0a88c5f6384f2b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`~list.append`, the content of the mutable object has +After the call to :meth:`!append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -1397,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`!list.extend`:: @@ -1903,7 +1903,7 @@ identity tests. This prevents the code from being confused by objects such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of -:meth:`collections.abc.Sequence.__contains__`:: +:meth:`!collections.abc.Sequence.__contains__`:: def __contains__(self, value): for v in self: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f7e454af62c87f..ac5b54b38f37cb 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -27,7 +27,6 @@ Doc/extending/newtypes.rst Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst -Doc/faq/programming.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From f904aa4e1f6943e5bd9a8a73cf762f063e6fa247 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 20 Aug 2023 18:01:10 +0200 Subject: [PATCH 209/598] Docs: document 'manager' and '_log' attrs of logging.Logging (#108145) Authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/logging.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 97a0b82b78b1b3..cf3dcb96a4c412 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1019,6 +1019,14 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. + .. attribute:: manager + + Delegates to the underlying :attr:`!manager`` on *logger*. + + .. attribute:: _log + + Delegates to the underlying :meth:`!_log`` method on *logger*. + In addition to the above, :class:`LoggerAdapter` supports the following methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, From 92815cc7cf3df8ab702c7cea4efaef349a4b0480 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 20 Aug 2023 19:34:27 +0100 Subject: [PATCH 210/598] Resolve reference warnings in faq/design.rst (#108148) --- Doc/faq/design.rst | 4 ++-- Doc/tools/.nitignore | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 11d01374dc1e79..ae02c443e5938b 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -584,9 +584,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`list.append` method is expected to add new elements +program. For example, the :meth:`!list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`list.append` implementation will actually do this correctly, but it's +your :meth:`!list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ac5b54b38f37cb..0262a0afe45461 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst Doc/glossary.rst From 6323bc33ff9f445a947adf4af42b8be7e44c730c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 20 Aug 2023 20:01:13 +0100 Subject: [PATCH 211/598] Resolve reference warnings in faq/library.rst (#108149) Co-authored-by: Erlend E. Aasland Co-authored-by: Hugo van Kemenade --- Doc/faq/library.rst | 13 ++++++++----- Doc/tools/.nitignore | 1 - 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 9e3727456bb96e..476a43d9c288f1 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python? ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +:c:func:`!onexit`. Why don't my signal handlers work? @@ -397,7 +397,7 @@ These aren't:: D[x] = D[x] + 1 Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can +:meth:`~object.__del__` method when their reference count reaches zero, and that can affect things. This is especially true for the mass updates to dictionaries and lists. When in doubt, use a mutex! @@ -730,14 +730,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`socket.connect`, you will either connect immediately +mode. Then when you do the :meth:`~socket.socket.connect`, +you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will -just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later +You can use the :meth:`~socket.socket.connect_ex` method +to avoid creating an exception. +It will just return the errno value. +To poll, you can call :meth:`~socket.socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this socket to :meth:`select.select` to check if it's writable. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0262a0afe45461..9475e883ee5b54 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -25,7 +25,6 @@ Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/faq/gui.rst -Doc/faq/library.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From 05ef4ca94c694bc50e6fd221fe648d05d227f3a4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 21 Aug 2023 00:40:41 +0100 Subject: [PATCH 212/598] gh-104504: cases generator: Add `--warn-unreachable` to the mypy config (#108112) --- Tools/cases_generator/_typing_backports.py | 15 ++++++++++ Tools/cases_generator/analysis.py | 7 +++-- Tools/cases_generator/generate_cases.py | 33 +++++++++------------- Tools/cases_generator/mypy.ini | 9 +++--- 4 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 Tools/cases_generator/_typing_backports.py diff --git a/Tools/cases_generator/_typing_backports.py b/Tools/cases_generator/_typing_backports.py new file mode 100644 index 00000000000000..c2aa50804cefe2 --- /dev/null +++ b/Tools/cases_generator/_typing_backports.py @@ -0,0 +1,15 @@ +"""Backports from newer versions of the typing module. + +We backport these features here so that Python can still build +while using an older Python version for PYTHON_FOR_REGEN. +""" + +from typing import NoReturn + + +def assert_never(obj: NoReturn) -> NoReturn: + """Statically assert that a line of code is unreachable. + + Backport of typing.assert_never (introduced in Python 3.11). + """ + raise AssertionError(f"Expected code to be unreachable, but got: {obj}") diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 48f2db981c95b6..72fb2d761dbfae 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -2,6 +2,7 @@ import sys import typing +from _typing_backports import assert_never from flags import InstructionFlags, variable_used from formatting import prettify_filename, UNUSED from instructions import ( @@ -172,7 +173,7 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: self.pseudos[name] = thing self.everything.append(thing) case _: - typing.assert_never(thing) + assert_never(thing) if not psr.eof(): raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") @@ -368,7 +369,7 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: # SAVE_IP in a macro is a no-op in Tier 1 flags.add(instr.instr_flags) case _: - typing.assert_never(component) + assert_never(component) format = "IB" if flags.HAS_ARG_FLAG else "IX" if offset: format += "C" + "0" * (offset - 1) @@ -409,5 +410,5 @@ def check_macro_components( case parsing.CacheEffect(): components.append(uop) case _: - typing.assert_never(uop) + assert_never(uop) return components diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index de31129ac0548d..7b1880b98a8feb 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -13,6 +13,7 @@ from collections.abc import Iterator import stacking # Early import to avoid circular import +from _typing_backports import assert_never from analysis import Analyzer from formatting import Formatter, list_effect_size from flags import InstructionFlags, variable_used @@ -146,7 +147,7 @@ class Generator(Analyzer): def get_stack_effect_info( self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo - ) -> tuple[AnyInstruction | None, str | None, str | None]: + ) -> tuple[AnyInstruction | None, str, str]: def effect_str(effects: list[StackEffect]) -> str: n_effect, sym_effect = list_effect_size(effects) if sym_effect: @@ -154,8 +155,6 @@ def effect_str(effects: list[StackEffect]) -> str: return str(n_effect) instr: AnyInstruction | None - popped: str | None - pushed: str | None match thing: case parsing.InstDef(): if thing.kind != "op" or self.instrs[thing.name].is_viable_uop(): @@ -171,10 +170,9 @@ def effect_str(effects: list[StackEffect]) -> str: popped, pushed = stacking.get_stack_effect_info_for_macro(instr) case parsing.Pseudo(): instr = self.pseudo_instrs[thing.name] - popped = pushed = None # Calculate stack effect, and check that it's the the same # for all targets. - for target in self.pseudos[thing.name].targets: + for idx, target in enumerate(self.pseudos[thing.name].targets): target_instr = self.instrs.get(target) # Currently target is always an instr. This could change # in the future, e.g., if we have a pseudo targetting a @@ -182,14 +180,13 @@ def effect_str(effects: list[StackEffect]) -> str: assert target_instr target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) - if pushed is None: - assert popped is None + if idx == 0: popped, pushed = target_popped, target_pushed else: assert popped == target_popped assert pushed == target_pushed case _: - typing.assert_never(thing) + assert_never(thing) return instr, popped, pushed @contextlib.contextmanager @@ -209,7 +206,6 @@ def write_stack_effect_functions(self) -> None: continue instr, popped, pushed = self.get_stack_effect_info(thing) if instr is not None: - assert popped is not None and pushed is not None popped_data.append((instr, popped)) pushed_data.append((instr, pushed)) @@ -379,7 +375,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Compute the set of all instruction formats. all_formats: set[str] = set() for thing in self.everything: - format: str | None match thing: case OverriddenInstructionPlaceHolder(): continue @@ -388,17 +383,15 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Macro(): format = self.macro_instrs[thing.name].instr_fmt case parsing.Pseudo(): - format = None - for target in self.pseudos[thing.name].targets: + for idx, target in enumerate(self.pseudos[thing.name].targets): target_instr = self.instrs.get(target) assert target_instr - if format is None: + if idx == 0: format = target_instr.instr_fmt else: assert format == target_instr.instr_fmt - assert format is not None case _: - typing.assert_never(thing) + assert_never(thing) all_formats.add(format) # Turn it into a sorted list of enum values. @@ -488,7 +481,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.pseudo_instrs[thing.name] ) case _: - typing.assert_never(thing) + assert_never(thing) with self.metadata_item( "const struct opcode_macro_expansion " @@ -525,7 +518,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) with self.metadata_item( "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" @@ -774,7 +767,7 @@ def write_instructions( case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) print( f"Wrote {n_instrs} instructions and {n_macros} macros " @@ -818,7 +811,7 @@ def write_executor_instructions( case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) print( f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}", file=sys.stderr, @@ -850,7 +843,7 @@ def write_abstract_interpreter_instructions( case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) print( f"Wrote some stuff to {abstract_interpreter_filename}", file=sys.stderr, diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index 7480841bf07edc..54ccb8c5c1a3a2 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -2,13 +2,12 @@ files = Tools/cases_generator/ pretty = True +# Make sure Python can still be built +# using Python 3.10 for `PYTHON_FOR_REGEN`... python_version = 3.10 -# Be strict: +# ...And be strict: strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool - -# Don't enable this one yet; -# it has a lot of false positives on `cases_generator` -warn_unreachable = False +warn_unreachable = True From db6dc6ce41f42ec2cb4fdfbc0a099114f42e888f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 21 Aug 2023 03:54:10 +0300 Subject: [PATCH 213/598] gh-107526: Revert "gh-100357: Convert several functions in bltinsmodule to AC" (#107542) --- ...-08-09-15-05-27.gh-issue-107526.PB32z-.rst | 2 + Python/bltinmodule.c | 206 +++++++++--------- Python/clinic/bltinmodule.c.h | 196 +---------------- 3 files changed, 101 insertions(+), 303 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst new file mode 100644 index 00000000000000..42ea09e78d41cc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst @@ -0,0 +1,2 @@ +Revert converting ``vars``, ``dir``, ``next``, ``getattr``, and ``iter`` to +argument clinic. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 9baf233614879e..d06efcf6ba3687 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -842,33 +842,31 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, return result; } -/*[clinic input] -dir as builtin_dir - - arg: object = NULL - / - -Show attributes of an object. - -If called without an argument, return the names in the current scope. -Else, return an alphabetized list of names comprising (some of) the attributes -of the given object, and of attributes reachable from it. -If the object supplies a method named __dir__, it will be used; otherwise -the default dir() logic is used and returns: - for a module object: the module's attributes. - for a class object: its attributes, and recursively the attributes - of its bases. - for any other object: its attributes, its class's attributes, and - recursively the attributes of its class's base classes. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_dir_impl(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=24f2c7a52c1e3b08 input=ed6d6ccb13d52251]*/ +builtin_dir(PyObject *self, PyObject *args) { + PyObject *arg = NULL; + + if (!PyArg_UnpackTuple(args, "dir", 0, 1, &arg)) + return NULL; return PyObject_Dir(arg); } +PyDoc_STRVAR(dir_doc, +"dir([object]) -> list of strings\n" +"\n" +"If called without an argument, return the names in the current scope.\n" +"Else, return an alphabetized list of names comprising (some of) the attributes\n" +"of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used; otherwise\n" +"the default dir() logic is used and returns:\n" +" for a module object: the module's attributes.\n" +" for a class object: its attributes, and recursively the attributes\n" +" of its bases.\n" +" for any other object: its attributes, its class's attributes, and\n" +" recursively the attributes of its class's base classes."); + /*[clinic input] divmod as builtin_divmod @@ -1138,39 +1136,36 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } -/*[clinic input] -getattr as builtin_getattr - - object: object - name: object - default: object = NULL - / - -Get a named attribute from an object. - -getattr(x, 'y') is equivalent to x.y -When a default argument is given, it is returned when the attribute doesn't -exist; without it, an exception is raised in that case. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, - PyObject *default_value) -/*[clinic end generated code: output=74ad0e225e3f701c input=d7562cd4c3556171]*/ +builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *result; + PyObject *v, *name, *result; + + if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) + return NULL; - if (default_value != NULL) { - if (PyObject_GetOptionalAttr(object, name, &result) == 0) { - return Py_NewRef(default_value); + v = args[0]; + name = args[1]; + if (nargs > 2) { + if (PyObject_GetOptionalAttr(v, name, &result) == 0) { + PyObject *dflt = args[2]; + return Py_NewRef(dflt); } } else { - result = PyObject_GetAttr(object, name); + result = PyObject_GetAttr(v, name); } return result; } +PyDoc_STRVAR(getattr_doc, +"getattr(object, name[, default]) -> value\n\ +\n\ +Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ +When a default argument is given, it is returned when the attribute doesn't\n\ +exist; without it, an exception is raised in that case."); + /*[clinic input] globals as builtin_globals @@ -1482,43 +1477,34 @@ PyTypeObject PyMap_Type = { }; -/*[clinic input] -next as builtin_next - - iterator: object - default: object = NULL - / - -Return the next item from the iterator. - -If default is given and the iterator is exhausted, -it is returned instead of raising StopIteration. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_next_impl(PyObject *module, PyObject *iterator, - PyObject *default_value) -/*[clinic end generated code: output=a38a94eeb447fef9 input=180f9984f182020f]*/ +builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *res; + PyObject *it, *res; + + if (!_PyArg_CheckPositional("next", nargs, 1, 2)) + return NULL; - if (!PyIter_Check(iterator)) { + it = args[0]; + if (!PyIter_Check(it)) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", - Py_TYPE(iterator)->tp_name); + Py_TYPE(it)->tp_name); return NULL; } - res = (*Py_TYPE(iterator)->tp_iternext)(iterator); + res = (*Py_TYPE(it)->tp_iternext)(it); if (res != NULL) { return res; - } else if (default_value != NULL) { + } else if (nargs > 1) { + PyObject *def = args[1]; if (PyErr_Occurred()) { if(!PyErr_ExceptionMatches(PyExc_StopIteration)) return NULL; PyErr_Clear(); } - return Py_NewRef(default_value); + return Py_NewRef(def); } else if (PyErr_Occurred()) { return NULL; } else { @@ -1527,6 +1513,12 @@ builtin_next_impl(PyObject *module, PyObject *iterator, } } +PyDoc_STRVAR(next_doc, +"next(iterator[, default])\n\ +\n\ +Return the next item from the iterator. If default is given and the iterator\n\ +is exhausted, it is returned instead of raising StopIteration."); + /*[clinic input] setattr as builtin_setattr @@ -1620,33 +1612,34 @@ builtin_hex(PyObject *module, PyObject *number) } -/*[clinic input] -iter as builtin_iter - - object: object - sentinel: object = NULL - / - -Get an iterator from an object. - -In the first form, the argument must supply its own iterator, or be a sequence. -In the second form, the callable is called until it returns the sentinel. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel) -/*[clinic end generated code: output=12cf64203c195a94 input=a5d64d9d81880ba6]*/ +builtin_iter(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - if (sentinel == NULL) - return PyObject_GetIter(object); - if (!PyCallable_Check(object)) { + PyObject *v; + + if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) + return NULL; + v = args[0]; + if (nargs == 1) + return PyObject_GetIter(v); + if (!PyCallable_Check(v)) { PyErr_SetString(PyExc_TypeError, - "iter(object, sentinel): object must be callable"); + "iter(v, w): v must be callable"); return NULL; } - return PyCallIter_New(object, sentinel); + PyObject *sentinel = args[1]; + return PyCallIter_New(v, sentinel); } +PyDoc_STRVAR(iter_doc, +"iter(iterable) -> iterator\n\ +iter(callable, sentinel) -> iterator\n\ +\n\ +Get an iterator from an object. In the first form, the argument must\n\ +supply its own iterator, or be a sequence.\n\ +In the second form, the callable is called until it returns the sentinel."); + /*[clinic input] aiter as builtin_aiter @@ -2444,29 +2437,20 @@ builtin_sorted(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject } -/*[clinic input] -vars as builtin_vars - - object: object = NULL - / - -Show vars. - -Without arguments, equivalent to locals(). -With an argument, equivalent to object.__dict__. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_vars_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=840a7f64007a3e0a input=80cbdef9182c4ba3]*/ +builtin_vars(PyObject *self, PyObject *args) { + PyObject *v = NULL; PyObject *d; - if (object == NULL) { + if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) + return NULL; + if (v == NULL) { d = _PyEval_GetFrameLocals(); } else { - if (PyObject_GetOptionalAttr(object, &_Py_ID(__dict__), &d) == 0) { + if (PyObject_GetOptionalAttr(v, &_Py_ID(__dict__), &d) == 0) { PyErr_SetString(PyExc_TypeError, "vars() argument must have __dict__ attribute"); } @@ -2474,6 +2458,12 @@ builtin_vars_impl(PyObject *module, PyObject *object) return d; } +PyDoc_STRVAR(vars_doc, +"vars([object]) -> dictionary\n\ +\n\ +Without arguments, equivalent to locals().\n\ +With an argument, equivalent to object.__dict__."); + /*[clinic input] sum as builtin_sum @@ -3022,12 +3012,12 @@ static PyMethodDef builtin_methods[] = { BUILTIN_CHR_METHODDEF BUILTIN_COMPILE_METHODDEF BUILTIN_DELATTR_METHODDEF - BUILTIN_DIR_METHODDEF + {"dir", builtin_dir, METH_VARARGS, dir_doc}, BUILTIN_DIVMOD_METHODDEF BUILTIN_EVAL_METHODDEF BUILTIN_EXEC_METHODDEF BUILTIN_FORMAT_METHODDEF - BUILTIN_GETATTR_METHODDEF + {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, getattr_doc}, BUILTIN_GLOBALS_METHODDEF BUILTIN_HASATTR_METHODDEF BUILTIN_HASH_METHODDEF @@ -3036,13 +3026,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - BUILTIN_ITER_METHODDEF + {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, iter_doc}, BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, - BUILTIN_NEXT_METHODDEF + {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF @@ -3053,7 +3043,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_SETATTR_METHODDEF BUILTIN_SORTED_METHODDEF BUILTIN_SUM_METHODDEF - BUILTIN_VARS_METHODDEF + {"vars", builtin_vars, METH_VARARGS, vars_doc}, {NULL, NULL}, }; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index b0d05dde956efd..7540de68280430 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -383,49 +383,6 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -PyDoc_STRVAR(builtin_dir__doc__, -"dir($module, arg=, /)\n" -"--\n" -"\n" -"Show attributes of an object.\n" -"\n" -"If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" -" for a module object: the module\'s attributes.\n" -" for a class object: its attributes, and recursively the attributes\n" -" of its bases.\n" -" for any other object: its attributes, its class\'s attributes, and\n" -" recursively the attributes of its class\'s base classes."); - -#define BUILTIN_DIR_METHODDEF \ - {"dir", _PyCFunction_CAST(builtin_dir), METH_FASTCALL, builtin_dir__doc__}, - -static PyObject * -builtin_dir_impl(PyObject *module, PyObject *arg); - -static PyObject * -builtin_dir(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *arg = NULL; - - if (!_PyArg_CheckPositional("dir", nargs, 0, 1)) { - goto exit; - } - if (nargs < 1) { - goto skip_optional; - } - arg = args[0]; -skip_optional: - return_value = builtin_dir_impl(module, arg); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_divmod__doc__, "divmod($module, x, y, /)\n" "--\n" @@ -586,47 +543,6 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject return return_value; } -PyDoc_STRVAR(builtin_getattr__doc__, -"getattr($module, object, name, default=, /)\n" -"--\n" -"\n" -"Get a named attribute from an object.\n" -"\n" -"getattr(x, \'y\') is equivalent to x.y\n" -"When a default argument is given, it is returned when the attribute doesn\'t\n" -"exist; without it, an exception is raised in that case."); - -#define BUILTIN_GETATTR_METHODDEF \ - {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, builtin_getattr__doc__}, - -static PyObject * -builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, - PyObject *default_value); - -static PyObject * -builtin_getattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *object; - PyObject *name; - PyObject *default_value = NULL; - - if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) { - goto exit; - } - object = args[0]; - name = args[1]; - if (nargs < 3) { - goto skip_optional; - } - default_value = args[2]; -skip_optional: - return_value = builtin_getattr_impl(module, object, name, default_value); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_globals__doc__, "globals($module, /)\n" "--\n" @@ -692,44 +608,6 @@ PyDoc_STRVAR(builtin_id__doc__, #define BUILTIN_ID_METHODDEF \ {"id", (PyCFunction)builtin_id, METH_O, builtin_id__doc__}, -PyDoc_STRVAR(builtin_next__doc__, -"next($module, iterator, default=, /)\n" -"--\n" -"\n" -"Return the next item from the iterator.\n" -"\n" -"If default is given and the iterator is exhausted,\n" -"it is returned instead of raising StopIteration."); - -#define BUILTIN_NEXT_METHODDEF \ - {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, builtin_next__doc__}, - -static PyObject * -builtin_next_impl(PyObject *module, PyObject *iterator, - PyObject *default_value); - -static PyObject * -builtin_next(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *iterator; - PyObject *default_value = NULL; - - if (!_PyArg_CheckPositional("next", nargs, 1, 2)) { - goto exit; - } - iterator = args[0]; - if (nargs < 2) { - goto skip_optional; - } - default_value = args[1]; -skip_optional: - return_value = builtin_next_impl(module, iterator, default_value); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_setattr__doc__, "setattr($module, obj, name, value, /)\n" "--\n" @@ -821,43 +699,6 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, -PyDoc_STRVAR(builtin_iter__doc__, -"iter($module, object, sentinel=, /)\n" -"--\n" -"\n" -"Get an iterator from an object.\n" -"\n" -"In the first form, the argument must supply its own iterator, or be a sequence.\n" -"In the second form, the callable is called until it returns the sentinel."); - -#define BUILTIN_ITER_METHODDEF \ - {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, builtin_iter__doc__}, - -static PyObject * -builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel); - -static PyObject * -builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *object; - PyObject *sentinel = NULL; - - if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) { - goto exit; - } - object = args[0]; - if (nargs < 2) { - goto skip_optional; - } - sentinel = args[1]; -skip_optional: - return_value = builtin_iter_impl(module, object, sentinel); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_aiter__doc__, "aiter($module, async_iterable, /)\n" "--\n" @@ -1236,41 +1077,6 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -PyDoc_STRVAR(builtin_vars__doc__, -"vars($module, object=, /)\n" -"--\n" -"\n" -"Show vars.\n" -"\n" -"Without arguments, equivalent to locals().\n" -"With an argument, equivalent to object.__dict__."); - -#define BUILTIN_VARS_METHODDEF \ - {"vars", _PyCFunction_CAST(builtin_vars), METH_FASTCALL, builtin_vars__doc__}, - -static PyObject * -builtin_vars_impl(PyObject *module, PyObject *object); - -static PyObject * -builtin_vars(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *object = NULL; - - if (!_PyArg_CheckPositional("vars", nargs, 0, 1)) { - goto exit; - } - if (nargs < 1) { - goto skip_optional; - } - object = args[0]; -skip_optional: - return_value = builtin_vars_impl(module, object); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_sum__doc__, "sum($module, iterable, /, start=0)\n" "--\n" @@ -1406,4 +1212,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=ef2f16ece134d62d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=daeee81b018824f4 input=a9049054013a1b77]*/ From 04f7875c4489bbe817e76f9f33773c8c21ba4ec2 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 21 Aug 2023 13:51:31 +0900 Subject: [PATCH 214/598] gh-107526: Fix test_module_level_callable_unrepresentable_default (gh-108187) --- Lib/test/test_pydoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index fe4e37d4858c85..9b5c11bf853fd9 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1232,7 +1232,7 @@ def test_bound_builtin_classmethod_o(self): def test_module_level_callable_unrepresentable_default(self): self.assertEqual(self._get_summary_line(getattr), - "getattr(object, name, default=, /)") + "getattr(...)") def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), From 4fdf3fda0f970805b2404e71f466f430ab8cd9fd Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 21 Aug 2023 14:50:09 +0900 Subject: [PATCH 215/598] gh-107265: Fix code_richcompare for ENTER_EXECUTOR case (gh-108165) --- Lib/test/test_capi/test_misc.py | 11 +++++++++++ Objects/codeobject.c | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 18a0476122dabf..ea0504333bab00 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2341,6 +2341,17 @@ def long_loop(): long_loop() self.assertEqual(opt.get_count(), 10) + def test_code_richcompare(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + self.assertEqual(testfunc.__code__, testfunc.__code__.replace()) + def get_first_executor(func): code = func.__code__ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 4d6efe938f45d6..c34905c3196063 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1781,8 +1781,25 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; + + if (co_instr.op.code == ENTER_EXECUTOR) { + const int exec_index = co_instr.op.arg; + _PyExecutorObject *exec = co->co_executors->executors[exec_index]; + co_instr.op.code = exec->vm_data.opcode; + co_instr.op.arg = exec->vm_data.oparg; + } + assert(co_instr.op.code != ENTER_EXECUTOR); co_instr.op.code = _PyOpcode_Deopt[co_instr.op.code]; + + if (cp_instr.op.code == ENTER_EXECUTOR) { + const int exec_index = cp_instr.op.arg; + _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; + cp_instr.op.code = exec->vm_data.opcode; + cp_instr.op.arg = exec->vm_data.oparg; + } + assert(cp_instr.op.code != ENTER_EXECUTOR); cp_instr.op.code = _PyOpcode_Deopt[cp_instr.op.code]; + eq = co_instr.cache == cp_instr.cache; if (!eq) { goto unequal; From 8f3d09bf5d16b508fece5420a22abe6f0c1f00b7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 21 Aug 2023 08:37:52 +0100 Subject: [PATCH 216/598] Resolve reference warnings in faq/gui.rst (#108147) Co-authored-by: Hugo van Kemenade --- Doc/faq/gui.rst | 9 +++++---- Doc/tools/.nitignore | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 0a372342862d2f..cfa60feceb31b7 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -43,7 +43,7 @@ applications, the applications will not be truly stand-alone, as the application will still need the Tcl and Tk libraries. One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY` +to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY` environment variables. Various third-party freeze libraries such as py2exe and cx_Freeze have @@ -55,7 +55,7 @@ Can I have Tk events handled while waiting for I/O? On platforms other than Windows, yes, and you don't even need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you +code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you to register a callback function which will be called from the Tk mainloop when I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. @@ -63,8 +63,9 @@ I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. I can't get key bindings to work in Tkinter: why? ------------------------------------------------- -An often-heard complaint is that event handlers bound to events with the -:meth:`bind` method don't get handled even when the appropriate key is pressed. +An often-heard complaint is that event handlers :ref:`bound ` +to events with the :meth:`!bind` method +don't get handled even when the appropriate key is pressed. The most common cause is that the widget to which the binding applies doesn't have "keyboard focus". Check out the Tk documentation for the focus command. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 9475e883ee5b54..1b43a2e4270c9c 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/gui.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From 014a5b71e7538926ae1c03c8c5ea13c96e741be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joon=20Hwan=20=EA=B9=80=EC=A4=80=ED=99=98?= Date: Mon, 21 Aug 2023 16:55:09 +0900 Subject: [PATCH 217/598] gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) --- Lib/test/test_asyncio/test_runners.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index b1eb6f492886b3..1eb5641914f2a4 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -101,11 +101,14 @@ async def main(expected): loop = asyncio.get_event_loop() self.assertIs(loop.get_debug(), expected) - asyncio.run(main(False)) + asyncio.run(main(False), debug=False) asyncio.run(main(True), debug=True) with mock.patch('asyncio.coroutines._is_debug_mode', lambda: True): asyncio.run(main(True)) asyncio.run(main(False), debug=False) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: False): + asyncio.run(main(True), debug=True) + asyncio.run(main(False)) def test_asyncio_run_from_running_loop(self): async def main(): From 20cc90c0df3e368fe7cb63d958f0b17a78fa9d0a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 21 Aug 2023 04:49:08 -0500 Subject: [PATCH 218/598] gh-105736: Sync pure python version of OrderedDict with the C version (#108098) --- Lib/collections/__init__.py | 16 +++++++++------- Lib/test/test_ordered_dict.py | 11 +++++++++++ ...023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst | 3 +++ 3 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 03ca2d7e18f6f0..8652dc8a4ec450 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -95,17 +95,19 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - try: - self.__root - except AttributeError: - self.__hardroot = _Link() - self.__root = root = _proxy(self.__hardroot) - root.prev = root.next = root - self.__map = {} self.__update(other, **kwds) def __setitem__(self, key, value, diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index decbcc2419c9fc..4571b23dfe7c1a 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -122,6 +122,17 @@ def items(self): self.OrderedDict(Spam()) self.assertEqual(calls, ['keys']) + def test_overridden_init(self): + # Sync-up pure Python OD class with C class where + # a consistent internal state is created in __new__ + # rather than __init__. + OrderedDict = self.OrderedDict + class ODNI(OrderedDict): + def __init__(*args, **kwargs): + pass + od = ODNI() + od['a'] = 1 # This used to fail because __init__ was bypassed + def test_fromkeys(self): OrderedDict = self.OrderedDict od = OrderedDict.fromkeys('abc') diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst new file mode 100644 index 00000000000000..f051317d141ff5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -0,0 +1,3 @@ +Harmonized the pure Python version of OrderedDict with the C version. Now, +both versions set up their internal state in `__new__`. Formerly, the pure +Python version did the set up in `__init__`. From 622ddc41674c2566062af82f7b079aa01d2aae8c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Aug 2023 13:41:01 +0300 Subject: [PATCH 219/598] Improve references in the tutorial (GH-108069) * Use full qualified names for references (even if they do not work now, they will work in future). * Silence references to examples. --- Doc/tools/.nitignore | 2 -- Doc/tutorial/classes.rst | 43 +++++++++++++++++---------------- Doc/tutorial/controlflow.rst | 2 +- Doc/tutorial/datastructures.rst | 12 ++++----- Doc/tutorial/inputoutput.rst | 8 +++--- Doc/tutorial/modules.rst | 30 +++++++++++------------ 6 files changed, 48 insertions(+), 49 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1b43a2e4270c9c..4231fb85758913 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -164,9 +164,7 @@ Doc/tutorial/appendix.rst Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst -Doc/tutorial/inputoutput.rst Doc/tutorial/introduction.rst -Doc/tutorial/modules.rst Doc/using/cmdline.rst Doc/using/configure.rst Doc/using/windows.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 06445e000c1ef6..91a3b73d2b55aa 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -91,7 +91,7 @@ Attributes may be read-only or writable. In the latter case, assignment to attributes is possible. Module attributes are writable: you can write ``modname.the_answer = 42``. Writable attributes may also be deleted with the :keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`the_answer` from the object named by ``modname``. +the attribute :attr:`!the_answer` from the object named by ``modname``. Namespaces are created at different moments and have different lifetimes. The namespace containing the built-in names is created when the Python interpreter @@ -249,7 +249,7 @@ created. This is basically a wrapper around the contents of the namespace created by the class definition; we'll learn more about class objects in the next section. The original local scope (the one in effect just before the class definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`ClassName` in the +class name given in the class definition header (:class:`!ClassName` in the example). @@ -291,20 +291,20 @@ variable ``x``. The instantiation operation ("calling" a class object) creates an empty object. Many classes like to create objects with instances customized to a specific initial state. Therefore a class may define a special method named -:meth:`__init__`, like this:: +:meth:`~object.__init__`, like this:: def __init__(self): self.data = [] -When a class defines an :meth:`__init__` method, class instantiation -automatically invokes :meth:`__init__` for the newly created class instance. So +When a class defines an :meth:`~object.__init__` method, class instantiation +automatically invokes :meth:`!__init__` for the newly created class instance. So in this example, a new, initialized instance can be obtained by:: x = MyClass() -Of course, the :meth:`__init__` method may have arguments for greater +Of course, the :meth:`~object.__init__` method may have arguments for greater flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`__init__`. For example, :: +are passed on to :meth:`!__init__`. For example, :: >>> class Complex: ... def __init__(self, realpart, imagpart): @@ -328,7 +328,7 @@ attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`MyClass` created above, the following piece of +``x`` is the instance of :class:`!MyClass` created above, the following piece of code will print the value ``16``, without leaving a trace:: x.counter = 1 @@ -363,7 +363,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`MyClass` example, this will return the string ``'hello world'``. +In the :class:`!MyClass` example, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -375,7 +375,7 @@ will continue to print ``hello world`` until the end of time. What exactly happens when a method is called? You may have noticed that ``x.f()`` was called without an argument above, even though the function -definition for :meth:`f` specified an argument. What happened to the argument? +definition for :meth:`!f` specified an argument. What happened to the argument? Surely Python raises an exception when a function that requires an argument is called without any --- even if the argument isn't actually used... @@ -532,9 +532,9 @@ variable in the class is also ok. For example:: h = g -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`C` that refer to +Now ``f``, ``g`` and ``h`` are all attributes of class :class:`!C` that refer to function objects, and consequently they are all methods of instances of -:class:`C` --- ``h`` being exactly equivalent to ``g``. Note that this practice +:class:`!C` --- ``h`` being exactly equivalent to ``g``. Note that this practice usually only serves to confuse the reader of a program. Methods may call other methods by using method attributes of the ``self`` @@ -581,7 +581,7 @@ this:: . -The name :class:`BaseClassName` must be defined in a +The name :class:`!BaseClassName` must be defined in a namespace accessible from the scope containing the derived class definition. In place of a base class name, other arbitrary expressions are also allowed. This can be useful, for example, when the base @@ -645,9 +645,9 @@ multiple base classes looks like this:: For most purposes, in the simplest cases, you can think of the search for attributes inherited from a parent class as depth-first, left-to-right, not searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`DerivedClassName`, it is searched -for in :class:`Base1`, then (recursively) in the base classes of :class:`Base1`, -and if it was not found there, it was searched for in :class:`Base2`, and so on. +Thus, if an attribute is not found in :class:`!DerivedClassName`, it is searched +for in :class:`!Base1`, then (recursively) in the base classes of :class:`!Base1`, +and if it was not found there, it was searched for in :class:`!Base2`, and so on. In fact, it is slightly more complex than that; the method resolution order changes dynamically to support cooperative calls to :func:`super`. This @@ -760,7 +760,8 @@ is to use :mod:`dataclasses` for this purpose:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the +can define a class with methods :meth:`~io.TextIOBase.read` and +:meth:`~io.TextIOBase.readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -769,7 +770,7 @@ data from a string buffer instead, and pass it as an argument. not cause the interpreter to read further input from it.) Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`m`, and ``m.__func__`` is the function object +object with the method :meth:`!m`, and ``m.__func__`` is the function object corresponding to the method. @@ -818,9 +819,9 @@ using the :func:`next` built-in function; this example shows how it all works:: StopIteration Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`__iter__` method which +iterator behavior to your classes. Define an :meth:`~container.__iter__` method which returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: +defines :meth:`!__next__`, then :meth:`!__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -879,7 +880,7 @@ easy to create:: Anything that can be done with generators can also be done with class-based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods +compact is that the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` methods are created automatically. Another key feature is that the local variables and execution state are diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 138d87f892e891..4bcc3768111ccd 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -534,7 +534,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index c8e89d9b79bddd..87614d082a1d4e 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -143,8 +143,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`append`. To retrieve an item from the -top of the stack, use :meth:`pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -341,7 +341,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -501,8 +501,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`append` and -:meth:`extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -567,7 +567,7 @@ Looping Techniques ================== When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`items` method. :: +retrieved at the same time using the :meth:`~dict.items` method. :: >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} >>> for k, v in knights.items(): diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index f5cdd84cbadefe..fe9ca9ccb9c7e0 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -15,7 +15,7 @@ Fancier Output Formatting ========================= So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`write` method +the :func:`print` function. (A third way is using the :meth:`~io.TextIOBase.write` method of file objects; the standard output file can be referenced as ``sys.stdout``. See the Library Reference for more information on this.) @@ -456,8 +456,8 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. -File objects have some additional methods, such as :meth:`~file.isatty` and -:meth:`~file.truncate` which are less frequently used; consult the Library +File objects have some additional methods, such as :meth:`~io.IOBase.isatty` and +:meth:`~io.IOBase.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. @@ -469,7 +469,7 @@ Saving structured data with :mod:`json` .. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`read` method only returns strings, which will have to +effort, since the :meth:`~io.TextIOBase.read` method only returns strings, which will have to be passed to a function like :func:`int`, which takes a string like ``'123'`` and returns its numeric value 123. When you want to save more complex data types like nested lists and dictionaries, parsing and serializing by hand diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 734dd1cfe6871a..bf9e8e0b7b8066 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -183,7 +183,7 @@ The Module Search Path .. index:: triple: module; search; path -When a module named :mod:`spam` is imported, the interpreter first searches for +When a module named :mod:`!spam` is imported, the interpreter first searches for a built-in module with that name. These module names are listed in :data:`sys.builtin_module_names`. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable @@ -389,7 +389,7 @@ Packages ======== Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`A.B` designates a submodule +module names". For example, the module name :mod:`!A.B` designates a submodule named ``B`` in a package named ``A``. Just like the use of modules saves the authors of different modules from having to worry about each other's global variable names, the use of dotted module names saves the authors of multi-module @@ -448,7 +448,7 @@ example:: import sound.effects.echo -This loads the submodule :mod:`sound.effects.echo`. It must be referenced with +This loads the submodule :mod:`!sound.effects.echo`. It must be referenced with its full name. :: sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) @@ -457,7 +457,7 @@ An alternative way of importing the submodule is:: from sound.effects import echo -This also loads the submodule :mod:`echo`, and makes it available without its +This also loads the submodule :mod:`!echo`, and makes it available without its package prefix, so it can be used as follows:: echo.echofilter(input, output, delay=0.7, atten=4) @@ -466,8 +466,8 @@ Yet another variation is to import the desired function or variable directly:: from sound.effects.echo import echofilter -Again, this loads the submodule :mod:`echo`, but this makes its function -:func:`echofilter` directly available:: +Again, this loads the submodule :mod:`!echo`, but this makes its function +:func:`!echofilter` directly available:: echofilter(input, output, delay=0.7, atten=4) @@ -510,7 +510,7 @@ code:: __all__ = ["echo", "surround", "reverse"] This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`sound.effects` package. +named submodules of the :mod:`!sound.effects` package. Be aware that submodules might become shadowed by locally defined names. For example, if you added a ``reverse`` function to the @@ -529,8 +529,8 @@ would only import the two submodules ``echo`` and ``surround``, but *not* the return msg[::-1] # in the case of a 'from sound.effects import *' If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`sound.effects` into the -current namespace; it only ensures that the package :mod:`sound.effects` has +does *not* import all submodules from the package :mod:`!sound.effects` into the +current namespace; it only ensures that the package :mod:`!sound.effects` has been imported (possibly running any initialization code in :file:`__init__.py`) and then imports whatever names are defined in the package. This includes any names defined (and submodules explicitly loaded) by :file:`__init__.py`. It @@ -541,8 +541,8 @@ previous :keyword:`import` statements. Consider this code:: import sound.effects.surround from sound.effects import * -In this example, the :mod:`echo` and :mod:`surround` modules are imported in the -current namespace because they are defined in the :mod:`sound.effects` package +In this example, the :mod:`!echo` and :mod:`!surround` modules are imported in the +current namespace because they are defined in the :mod:`!sound.effects` package when the ``from...import`` statement is executed. (This also works when ``__all__`` is defined.) @@ -561,15 +561,15 @@ packages. Intra-package References ------------------------ -When packages are structured into subpackages (as with the :mod:`sound` package +When packages are structured into subpackages (as with the :mod:`!sound` package in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`sound.filters.vocoder` needs to use -the :mod:`echo` module in the :mod:`sound.effects` package, it can use ``from +packages. For example, if the module :mod:`!sound.filters.vocoder` needs to use +the :mod:`!echo` module in the :mod:`!sound.effects` package, it can use ``from sound.effects import echo``. You can also write relative imports, with the ``from module import name`` form of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`surround` +parent packages involved in the relative import. From the :mod:`!surround` module for example, you might use:: from . import echo From acbd3f9c5c5f23e95267714e41236140d84fe962 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 21 Aug 2023 12:56:46 +0200 Subject: [PATCH 220/598] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Victor Stinner Co-authored-by: Lumír 'Frenzy' Balhar --- Doc/library/tarfile.rst | 5 + Lib/tarfile.py | 11 +- Lib/test/test_tarfile.py | 146 +++++++++++++++++- ...-08-10-17-36-22.gh-issue-107845.dABiMJ.rst | 3 + 4 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 00f3070324ec1e..62d67bc577c7d0 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -740,6 +740,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index df4e41f7a0d23a..1147d59d3d5104 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -742,7 +742,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -802,7 +802,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2eda7fc4ceac71..2cdf2c1fc2f297 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3337,10 +3337,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3416,7 +3418,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3445,6 +3448,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3491,8 +3496,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if os_helper.can_symlink(): @@ -3534,9 +3546,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3550,6 +3599,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if os_helper.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3560,10 +3610,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if os_helper.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') @symlink_test def test_absolute_symlink(self): @@ -3593,12 +3657,30 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") @symlink_test def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3649,6 +3731,56 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + @symlink_test + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 00000000000000..32c1fb93f4ab2c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From 80bdebdd8593f007a2232ec04a7729bba6ebf12c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Aug 2023 14:16:31 +0300 Subject: [PATCH 221/598] gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) --- .../C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst | 4 ++++ Python/errors.c | 8 ++++++++ 2 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst diff --git a/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst new file mode 100644 index 00000000000000..f1f16609b118ba --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst @@ -0,0 +1,4 @@ +C API functions :c:func:`PyErr_SetFromErrnoWithFilename`, +:c:func:`PyErr_SetExcFromWindowsErrWithFilename` and +:c:func:`PyErr_SetFromWindowsErrWithFilename` save now the error code before +calling :c:func:`PyUnicode_DecodeFSDefault`. diff --git a/Python/errors.c b/Python/errors.c index d9086c507251e6..fb5b3ff2c7ba51 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -895,10 +895,12 @@ PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { PyObject *name = NULL; if (filename) { + int i = errno; name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; } + errno = i; } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); @@ -998,6 +1000,9 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; @@ -1028,6 +1033,9 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; From 37135d25e269ede92bc7da363bebfa574782e59a Mon Sep 17 00:00:00 2001 From: balmeida-nokia <83089745+balmeida-nokia@users.noreply.github.com> Date: Mon, 21 Aug 2023 12:39:06 +0100 Subject: [PATCH 222/598] gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) In the stack call of: _init_read_gz() ``` _read, tarfile.py:548 read, tarfile.py:526 _init_read_gz, tarfile.py:491 ``` a try;except exists that uses `self.exception`, so it needs to be set before calling _init_read_gz(). --- Lib/tarfile.py | 2 +- Lib/test/test_tarfile.py | 17 +++++++++++++++++ ...23-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 1147d59d3d5104..a835d00c90c92c 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -372,8 +372,8 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": - self._init_read_gz() self.exception = zlib.error + self._init_read_gz() else: self._init_write_gz(compresslevel) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2cdf2c1fc2f297..0d6ca4315cfda4 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -938,6 +938,23 @@ class LzmaDetectReadTest(LzmaTest, DetectReadTest): pass +class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase): + """ + See: https://github.com/python/cpython/issues/107396 + """ + def runTest(self): + f = io.BytesIO( + b'\x1f\x8b' # header + b'\x08' # compression method + b'\x04' # flags + b'\0\0\0\0\0\0' # timestamp, compression data, OS ID + b'\0\x01' # size + b'\0\0\0\0\0' # corrupt data (zeros) + ) + with self.assertRaises(tarfile.ReadError): + tarfile.open(fileobj=f, mode='r|gz') + + class MemberReadTest(ReadTest, unittest.TestCase): def _test_member(self, tarinfo, chksum=None, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst new file mode 100644 index 00000000000000..73bff4bdbe024d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst @@ -0,0 +1 @@ +tarfiles; Fixed use before assignment of self.exception for gzip decompression From 71962e5237bb7e12d8fc1447b8ed8ba082e23f77 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 21 Aug 2023 16:13:55 +0300 Subject: [PATCH 223/598] Run sphinx-lint on Misc/NEWS.d/next/ (#108212) Co-authored-by: Alex Waygood --- .pre-commit-config.yaml | 2 +- .../Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst | 2 +- .../2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst | 2 +- .../2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst | 3 +-- .../2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst | 2 +- .../2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst | 2 +- .../next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst | 4 ++-- .../Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst | 2 +- .../Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst | 2 +- .../Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst | 2 +- .../Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst | 2 +- .../Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst | 2 +- .../Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst | 3 ++- .../Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst | 6 +++--- 14 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85a6de4abe0146..451cbe8bc84820 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,5 +14,5 @@ repos: hooks: - id: sphinx-lint args: [--enable=default-role] - files: ^Doc/ + files: ^Doc/|^Misc/NEWS.d/next/ types: [rst] diff --git a/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst b/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst index 40b2609e95c7e9..7febf99c48a79b 100644 --- a/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst +++ b/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst @@ -1 +1 @@ -Check for `linux/limits.h` before including it in `Modules/posixmodule.c`. +Check for ``linux/limits.h`` before including it in ``Modules/posixmodule.c``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst index 27d0e9929794f4..57a30f601379c2 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst @@ -1 +1 @@ -:opcode:`LOAD_CLOSURE` is now a pseudo-op. \ No newline at end of file +:opcode:`LOAD_CLOSURE` is now a pseudo-op. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst index faf43bd0c2f48e..0d074ffa9942d4 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst @@ -1,3 +1,2 @@ Specialize :opcode:`LOAD_ATTR` for non-descriptors on the class. Adds -:opcode:`LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES` and :opcode -`LOAD_ATTR_NONDESCRIPTOR_NO_DICT`. +:opcode:`LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES` and :opcode:`LOAD_ATTR_NONDESCRIPTOR_NO_DICT`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst index f5a0e539e5d05f..60d3304a3fab93 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst @@ -1 +1 @@ -Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. \ No newline at end of file +Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst index e47927b4e11886..a9ab5cd43f0ffb 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst @@ -1,4 +1,4 @@ Fix potential unaligned memory access on C APIs involving returned sequences -of `char *` pointers within the :mod:`grp` and :mod:`socket` modules. These +of ``char *`` pointers within the :mod:`grp` and :mod:`socket` modules. These were revealed using a ``-fsaniziter=alignment`` build on ARM macOS. Patch by Christopher Chavez. diff --git a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst index a1a4cf6d63725a..94d7cc9deadbb1 100644 --- a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst +++ b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst @@ -1,2 +1,2 @@ -Ensure `gettext(msg)` retrieve translations even if a plural form exists. In -other words: `gettext(msg) == ngettext(msg, '', 1)`. +Ensure ``gettext(msg)`` retrieve translations even if a plural form exists. In +other words: ``gettext(msg) == ngettext(msg, '', 1)``. diff --git a/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst b/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst index d2687154a58594..0c24ff70334e17 100644 --- a/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst +++ b/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst @@ -1 +1 @@ -Added `asyncio.taskgroups.__all__` to `asyncio.__all__` for export in star imports. +Added ``asyncio.taskgroups.__all__`` to ``asyncio.__all__`` for export in star imports. diff --git a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst index 23763818d84ba5..d86a6bfdabbfb0 100644 --- a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst +++ b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst @@ -1,2 +1,2 @@ Fix crash when calling ``repr`` with a manually constructed SignalDict object. -Patch by Charlie Zhao. \ No newline at end of file +Patch by Charlie Zhao. diff --git a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst index bbc53d76decbc3..bcda64153413df 100644 --- a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst +++ b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst @@ -1,4 +1,4 @@ -Fixed several bugs in zipfile.Path, including: in ``Path.match`, Windows +Fixed several bugs in zipfile.Path, including: in :meth:`zipfile.Path.match`, Windows separators are no longer honored (and never were meant to be); Fixed ``name``/``suffix``/``suffixes``/``stem`` operations when no filename is present and the Path is not at the root of the zipfile; Reworked glob for diff --git a/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst b/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst index 532f8059740daf..b9cf35adf95523 100644 --- a/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst +++ b/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst @@ -1 +1 @@ -Remove import of :mod:``pprint`` from :mod:``sysconfig``. +Remove import of :mod:`pprint` from :mod:`sysconfig`. diff --git a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst index 9d5ba1a2d7ccba..3c29ddaa634ca8 100644 --- a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst +++ b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst @@ -1,2 +1,2 @@ Shelves opened with :func:`shelve.open` have a much faster :meth:`clear` -method. Patch by James Cave. \ No newline at end of file +method. Patch by James Cave. diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst index cd2a5d0d5324a2..deea9af01c4ebd 100644 --- a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst @@ -1 +1,2 @@ -Fix `doctest.DocTestFinder.find` in presence of class names with special characters. Patch by Gertjan van Zwieten. +Fix :meth:`doctest.DocTestFinder.find` in presence of class names with special +characters. Patch by Gertjan van Zwieten. diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst index f051317d141ff5..1d959a3b22284c 100644 --- a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -1,3 +1,3 @@ -Harmonized the pure Python version of OrderedDict with the C version. Now, -both versions set up their internal state in `__new__`. Formerly, the pure -Python version did the set up in `__init__`. +Harmonized the pure Python version of :class:`~collections.OrderedDict` with the C version. Now, +both versions set up their internal state in ``__new__``. Formerly, the pure +Python version did the set up in ``__init__``. From 13104f3b7412dce9bf7cfd09bf2d6dad1f3cc2ed Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 21 Aug 2023 16:52:37 +0300 Subject: [PATCH 224/598] gh-107905: Test raising `__value__` for `TypeAliasType` (#107997) --- Lib/test/test_type_aliases.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 0ce97f57de6860..8f0a998e1f3dc1 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -168,6 +168,24 @@ def test_recursive_repr(self): self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), "GenericRecursive[GenericRecursive[int]]") + def test_raising(self): + type MissingName = list[_My_X] + with self.assertRaisesRegex( + NameError, + "cannot access free variable '_My_X' where it is not associated with a value", + ): + MissingName.__value__ + _My_X = int + self.assertEqual(MissingName.__value__, list[int]) + del _My_X + # Cache should still work: + self.assertEqual(MissingName.__value__, list[int]) + + # Explicit exception: + type ExprException = 1 / 0 + with self.assertRaises(ZeroDivisionError): + ExprException.__value__ + class TypeAliasConstructorTest(unittest.TestCase): def test_basic(self): From 60942cccb18cfd43240c1a1eb5deab7b31fcb81c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Aug 2023 16:59:58 +0300 Subject: [PATCH 225/598] gh-95065, gh-107704: Argument Clinic: support multiple '/ [from ...]' and '* [from ...]' markers (GH-108132) --- .../pycore_global_objects_fini_generated.h | 3 + Include/internal/pycore_global_strings.h | 3 + .../internal/pycore_runtime_init_generated.h | 3 + .../internal/pycore_unicodeobject_generated.h | 9 + Lib/test/test_clinic.py | 96 ++++- Modules/_testclinic.c | 82 ++++ Modules/clinic/_testclinic_depr.c.h | 363 ++++++++++++++++-- Tools/clinic/clinic.py | 110 +++--- 8 files changed, 565 insertions(+), 104 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ee9010583ff8b5..2f930babf6aad4 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -923,6 +923,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); @@ -954,6 +955,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); @@ -967,6 +969,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b081c0e023fa4a..5a0cd1a02ba561 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -412,6 +412,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) + STRUCT_FOR_ID(f) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(false) @@ -443,6 +444,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fset) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(g) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) @@ -456,6 +458,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(globals) STRUCT_FOR_ID(groupindex) STRUCT_FOR_ID(groups) + STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 8c9c7f753d8579..8c0fcdb1bdae1c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -918,6 +918,7 @@ extern "C" { INIT_ID(exp), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ + INIT_ID(f), \ INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(false), \ @@ -949,6 +950,7 @@ extern "C" { INIT_ID(fset), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(g), \ INIT_ID(generation), \ INIT_ID(genexpr), \ INIT_ID(get), \ @@ -962,6 +964,7 @@ extern "C" { INIT_ID(globals), \ INIT_ID(groupindex), \ INIT_ID(groups), \ + INIT_ID(h), \ INIT_ID(handle), \ INIT_ID(hash_name), \ INIT_ID(header), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 59f40075f93983..841eb787011271 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1077,6 +1077,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(extra_tokens); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(f); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(facility); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1170,6 +1173,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(future); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(g); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(generation); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1209,6 +1215,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(groups); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(h); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(handle); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 934c1e3ffccca8..310615083778f7 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1751,7 +1751,7 @@ def test_depr_star_must_come_before_star(self): * [from 3.14] Docstring. """ - err = "Function 'bar': '* [from ...]' must come before '*'" + err = "Function 'bar': '* [from ...]' must precede '*'" self.expect_failure(block, err, lineno=4) def test_depr_star_duplicate(self): @@ -1765,7 +1765,7 @@ def test_depr_star_duplicate(self): c: int Docstring. """ - err = "Function 'bar' uses '* [from ...]' more than once." + err = "Function 'bar' uses '* [from 3.14]' more than once." self.expect_failure(block, err, lineno=5) def test_depr_star_duplicate2(self): @@ -1779,7 +1779,7 @@ def test_depr_star_duplicate2(self): c: int Docstring. """ - err = "Function 'bar' uses '* [from ...]' more than once." + err = "Function 'bar': '* [from 3.15]' must precede '* [from 3.14]'" self.expect_failure(block, err, lineno=5) def test_depr_slash_duplicate(self): @@ -1793,7 +1793,7 @@ def test_depr_slash_duplicate(self): c: int Docstring. """ - err = "Function 'bar' uses '/ [from ...]' more than once." + err = "Function 'bar' uses '/ [from 3.14]' more than once." self.expect_failure(block, err, lineno=5) def test_depr_slash_duplicate2(self): @@ -1801,13 +1801,13 @@ def test_depr_slash_duplicate2(self): module foo foo.bar a: int - / [from 3.14] - b: int / [from 3.15] + b: int + / [from 3.14] c: int Docstring. """ - err = "Function 'bar' uses '/ [from ...]' more than once." + err = "Function 'bar': '/ [from 3.14]' must precede '/ [from 3.15]'" self.expect_failure(block, err, lineno=5) def test_single_slash(self): @@ -2724,7 +2724,15 @@ class ClinicFunctionalTest(unittest.TestCase): locals().update((name, getattr(ac_tester, name)) for name in dir(ac_tester) if name.startswith('test_')) - def check_depr_star(self, pnames, fn, *args, name=None, **kwds): + def check_depr(self, regex, fn, /, *args, **kwds): + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + + def check_depr_star(self, pnames, fn, /, *args, name=None, **kwds): if name is None: name = fn.__qualname__ if isinstance(fn, type): @@ -2734,12 +2742,7 @@ def check_depr_star(self, pnames, fn, *args, name=None, **kwds): fr"{re.escape(name)}\(\) is deprecated. Parameters? {pnames} will " fr"become( a)? keyword-only parameters? in Python 3\.14" ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - # Record the line number, so we're sure we've got the correct stack - # level on the deprecation warning. - _, lineno = fn(*args, **kwds), sys._getframe().f_lineno - self.assertEqual(cm.filename, __file__) - self.assertEqual(cm.lineno, lineno) + self.check_depr(regex, fn, *args, **kwds) def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): if name is None: @@ -2749,15 +2752,10 @@ def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): pl = 's' if ' ' in pnames else '' regex = ( fr"Passing keyword argument{pl} {pnames} to " - fr"{re.escape(name)}\(\) is deprecated. Corresponding parameter{pl} " + fr"{re.escape(name)}\(\) is deprecated. Parameter{pl} {pnames} " fr"will become positional-only in Python 3\.14." ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - # Record the line number, so we're sure we've got the correct stack - # level on the deprecation warning. - _, lineno = fn(*args, **kwds), sys._getframe().f_lineno - self.assertEqual(cm.filename, __file__) - self.assertEqual(cm.lineno, lineno) + self.check_depr(regex, fn, *args, **kwds) def test_objects_converter(self): with self.assertRaises(TypeError): @@ -3368,6 +3366,24 @@ def test_depr_star_noinline(self): check("a", "b", c="c") self.assertRaises(TypeError, fn, "a", "b", "c", "d") + def test_depr_star_multi(self): + fn = ac_tester.depr_star_multi + self.assertRaises(TypeError, fn, "a") + fn("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + errmsg = ( + "Passing more than 1 positional argument to depr_star_multi() is deprecated. " + "Parameter 'b' will become a keyword-only parameter in Python 3.16. " + "Parameters 'c' and 'd' will become keyword-only parameters in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become keyword-only parameters in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", "g", h="h") + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g", "h") + def test_depr_kwd_required_1(self): fn = ac_tester.depr_kwd_required_1 fn("a", "b") @@ -3452,6 +3468,44 @@ def test_depr_kwd_noinline(self): self.assertRaises(TypeError, fn, "a", c="c") self.assertRaises(TypeError, fn, a="a", b="b", c="c") + def test_depr_kwd_multi(self): + fn = ac_tester.depr_kwd_multi + fn("a", "b", "c", "d", "e", "f", "g", h="h") + errmsg = ( + "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to depr_kwd_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameters 'c' and 'd' will become positional-only in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become positional-only in Python 3.16.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + + def test_depr_multi(self): + fn = ac_tester.depr_multi + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g") + errmsg = ( + "Passing more than 4 positional arguments to depr_multi() is deprecated. " + "Parameter 'e' will become a keyword-only parameter in Python 3.15. " + "Parameter 'f' will become a keyword-only parameter in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g") + check("a", "b", "c", "d", "e", f="f", g="g") + fn("a", "b", "c", "d", e="e", f="f", g="g") + fn("a", "b", "c", d="d", e="e", f="f", g="g") + errmsg = ( + "Passing keyword arguments 'b' and 'c' to depr_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameter 'c' will become positional-only in Python 3.15.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g") + check("a", b="b", c="c", d="d", e="e", f="f", g="g") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index efec04d99029bb..2e0535d52641e0 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1580,6 +1580,32 @@ depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, } +/*[clinic input] +depr_star_multi + a: object + * [from 3.16] + b: object + * [from 3.15] + c: object + d: object + * [from 3.14] + e: object + f: object + g: object + * + h: object +[clinic start generated code]*/ + +static PyObject * +depr_star_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h) +/*[clinic end generated code: output=77681653f4202068 input=3ebd05d888a957ea]*/ +{ + Py_RETURN_NONE; +} + + /*[clinic input] depr_kwd_required_1 a: object @@ -1702,6 +1728,59 @@ depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, Py_RETURN_NONE; } + +/*[clinic input] +depr_kwd_multi + a: object + / + b: object + / [from 3.14] + c: object + d: object + / [from 3.15] + e: object + f: object + g: object + / [from 3.16] + h: object +[clinic start generated code]*/ + +static PyObject * +depr_kwd_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h) +/*[clinic end generated code: output=ddfbde80fe1942e1 input=7a074e621c79efd7]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_multi + a: object + / + b: object + / [from 3.14] + c: object + / [from 3.15] + d: object + * [from 3.15] + e: object + * [from 3.14] + f: object + * + g: object +[clinic start generated code]*/ + +static PyObject * +depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g) +/*[clinic end generated code: output=f81c92852ca2d4ee input=5b847c5e44bedd02]*/ +{ + Py_RETURN_NONE; +} + + // Reset PY_VERSION_HEX #undef PY_VERSION_HEX #define PY_VERSION_HEX _SAVED_PY_VERSION @@ -1779,6 +1858,7 @@ static PyMethodDef tester_methods[] = { DEPR_STAR_POS2_LEN2_METHODDEF DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF DEPR_STAR_NOINLINE_METHODDEF + DEPR_STAR_MULTI_METHODDEF DEPR_KWD_REQUIRED_1_METHODDEF DEPR_KWD_REQUIRED_2_METHODDEF DEPR_KWD_OPTIONAL_1_METHODDEF @@ -1786,6 +1866,8 @@ static PyMethodDef tester_methods[] = { DEPR_KWD_OPTIONAL_3_METHODDEF DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF DEPR_KWD_NOINLINE_METHODDEF + DEPR_KWD_MULTI_METHODDEF + DEPR_MULTI_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 661fdaf2a07181..b8365bd559b4e7 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -419,8 +419,7 @@ PyDoc_STRVAR(depr_kwd_new__doc__, "The deprecation message should use the class name instead of __new__.\n" "\n" "Note: Passing keyword argument \'a\' to _testclinic.DeprKwdNew() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'a\' will become positional-only in Python 3.14.\n" ""); static PyObject * @@ -479,8 +478,8 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'a' to _testclinic.DeprKwdNew() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'a' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -503,8 +502,7 @@ PyDoc_STRVAR(depr_kwd_init__doc__, "The deprecation message should use the class name instead of __init__.\n" "\n" "Note: Passing keyword argument \'a\' to _testclinic.DeprKwdInit() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'a\' will become positional-only in Python 3.14.\n" ""); static int @@ -563,8 +561,8 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'a' to _testclinic.DeprKwdInit() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'a' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -641,8 +639,8 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) } if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to " - "_testclinic.DeprKwdInitNoInline() is deprecated. Corresponding " - "parameters will become positional-only in Python 3.14.", 1)) + "_testclinic.DeprKwdInitNoInline() is deprecated. Parameters 'b' " + "and 'c' will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -1482,13 +1480,110 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +PyDoc_STRVAR(depr_star_multi__doc__, +"depr_star_multi($module, /, a, b, c, d, e, f, g, *, h)\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to depr_star_multi() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.16. Parameters \'c\' and \'d\' will become keyword-only\n" +"parameters in Python 3.15. Parameters \'e\', \'f\' and \'g\' will become\n" +"keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_MULTI_METHODDEF \ + {"depr_star_multi", _PyCFunction_CAST(depr_star_multi), METH_FASTCALL|METH_KEYWORDS, depr_star_multi__doc__}, + +static PyObject * +depr_star_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_multi'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_multi'.") +# else +# warning "Update the clinic input of 'depr_star_multi'." +# endif +#endif + +static PyObject * +depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 8 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", "f", "g", "h", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_multi", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[8]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + PyObject *f; + PyObject *g; + PyObject *h; + + if (nargs > 1 && nargs <= 7) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to depr_star_multi() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.16. Parameters 'c' and 'd' will become keyword-only " + "parameters in Python 3.15. Parameters 'e', 'f' and 'g' will " + "become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 7, 7, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + f = args[5]; + g = args[6]; + h = args[7]; + return_value = depr_star_multi_impl(module, a, b, c, d, e, f, g, h); + +exit: + return return_value; +} + PyDoc_STRVAR(depr_kwd_required_1__doc__, "depr_kwd_required_1($module, a, /, b)\n" "--\n" "\n" "Note: Passing keyword argument \'b\' to depr_kwd_required_1() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_REQUIRED_1_METHODDEF \ @@ -1548,8 +1643,8 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (nargs < 2) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'b' to depr_kwd_required_1() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'b' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -1567,7 +1662,7 @@ PyDoc_STRVAR(depr_kwd_required_2__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_required_2()\n" -"is deprecated. Corresponding parameters will become positional-only in\n" +"is deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" "Python 3.14.\n" ""); @@ -1630,7 +1725,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (nargs < 3) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_required_2() " - "is deprecated. Corresponding parameters will become " + "is deprecated. Parameters 'b' and 'c' will become " "positional-only in Python 3.14.", 1)) { goto exit; @@ -1650,8 +1745,7 @@ PyDoc_STRVAR(depr_kwd_optional_1__doc__, "--\n" "\n" "Note: Passing keyword argument \'b\' to depr_kwd_optional_1() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_OPTIONAL_1_METHODDEF \ @@ -1712,8 +1806,8 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (kwnames && PyTuple_GET_SIZE(kwnames) && nargs < 2 && args[1]) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'b' to depr_kwd_optional_1() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'b' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -1735,7 +1829,7 @@ PyDoc_STRVAR(depr_kwd_optional_2__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_optional_2()\n" -"is deprecated. Corresponding parameters will become positional-only in\n" +"is deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" "Python 3.14.\n" ""); @@ -1799,7 +1893,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_optional_2() " - "is deprecated. Corresponding parameters will become " + "is deprecated. Parameters 'b' and 'c' will become " "positional-only in Python 3.14.", 1)) { goto exit; @@ -1828,7 +1922,7 @@ PyDoc_STRVAR(depr_kwd_optional_3__doc__, "--\n" "\n" "Note: Passing keyword arguments \'a\', \'b\' and \'c\' to\n" -"depr_kwd_optional_3() is deprecated. Corresponding parameters will\n" +"depr_kwd_optional_3() is deprecated. Parameters \'a\', \'b\' and \'c\' will\n" "become positional-only in Python 3.14.\n" ""); @@ -1892,8 +1986,8 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 1 && args[0]) || (nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'a', 'b' and 'c' to " - "depr_kwd_optional_3() is deprecated. Corresponding parameters " - "will become positional-only in Python 3.14.", 1)) + "depr_kwd_optional_3() is deprecated. Parameters 'a', 'b' and 'c'" + " will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -1926,7 +2020,7 @@ PyDoc_STRVAR(depr_kwd_required_optional__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to\n" -"depr_kwd_required_optional() is deprecated. Corresponding parameters\n" +"depr_kwd_required_optional() is deprecated. Parameters \'b\' and \'c\'\n" "will become positional-only in Python 3.14.\n" ""); @@ -1990,8 +2084,8 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && args[2]))) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to " - "depr_kwd_required_optional() is deprecated. Corresponding " - "parameters will become positional-only in Python 3.14.", 1)) + "depr_kwd_required_optional() is deprecated. Parameters 'b' and " + "'c' will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -2014,7 +2108,7 @@ PyDoc_STRVAR(depr_kwd_noinline__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_noinline() is\n" -"deprecated. Corresponding parameters will become positional-only in\n" +"deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" "Python 3.14.\n" ""); @@ -2081,8 +2175,8 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO } if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_noinline() is " - "deprecated. Corresponding parameters will become positional-only" - " in Python 3.14.", 1)) + "deprecated. Parameters 'b' and 'c' will become positional-only " + "in Python 3.14.", 1)) { goto exit; } @@ -2092,4 +2186,209 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO exit: return return_value; } -/*[clinic end generated code: output=fc558c1efdcab076 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(depr_kwd_multi__doc__, +"depr_kwd_multi($module, a, /, b, c, d, e, f, g, h)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\', \'c\', \'d\', \'e\', \'f\' and \'g\' to\n" +"depr_kwd_multi() is deprecated. Parameter \'b\' will become positional-\n" +"only in Python 3.14. Parameters \'c\' and \'d\' will become positional-\n" +"only in Python 3.15. Parameters \'e\', \'f\' and \'g\' will become\n" +"positional-only in Python 3.16.\n" +""); + +#define DEPR_KWD_MULTI_METHODDEF \ + {"depr_kwd_multi", _PyCFunction_CAST(depr_kwd_multi), METH_FASTCALL|METH_KEYWORDS, depr_kwd_multi__doc__}, + +static PyObject * +depr_kwd_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_multi'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_multi'.") +# else +# warning "Update the clinic input of 'depr_kwd_multi'." +# endif +#endif + +static PyObject * +depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 7 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", "e", "f", "g", "h", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_multi", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[8]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + PyObject *f; + PyObject *g; + PyObject *h; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 8, 8, 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 7) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to " + "depr_kwd_multi() is deprecated. Parameter 'b' will become " + "positional-only in Python 3.14. Parameters 'c' and 'd' will " + "become positional-only in Python 3.15. Parameters 'e', 'f' and " + "'g' will become positional-only in Python 3.16.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + f = args[5]; + g = args[6]; + h = args[7]; + return_value = depr_kwd_multi_impl(module, a, b, c, d, e, f, g, h); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_multi__doc__, +"depr_multi($module, a, /, b, c, d, e, f, *, g)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_multi() is\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" +"Parameter \'c\' will become positional-only in Python 3.15.\n" +"\n" +"\n" +"Note: Passing more than 4 positional arguments to depr_multi() is\n" +"deprecated. Parameter \'e\' will become a keyword-only parameter in\n" +"Python 3.15. Parameter \'f\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_MULTI_METHODDEF \ + {"depr_multi", _PyCFunction_CAST(depr_multi), METH_FASTCALL|METH_KEYWORDS, depr_multi__doc__}, + +static PyObject * +depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_multi'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_multi'.") +# else +# warning "Update the clinic input of 'depr_multi'." +# endif +#endif + +static PyObject * +depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", "e", "f", "g", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_multi", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[7]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + PyObject *f; + PyObject *g; + + if (nargs > 4 && nargs <= 6) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 4 positional arguments to depr_multi() is " + "deprecated. Parameter 'e' will become a keyword-only parameter " + "in Python 3.15. Parameter 'f' will become a keyword-only " + "parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 1, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_multi() is " + "deprecated. Parameter 'b' will become positional-only in Python " + "3.14. Parameter 'c' will become positional-only in Python 3.15.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + f = args[5]; + g = args[6]; + return_value = depr_multi_impl(module, a, b, c, d, e, f, g); + +exit: + return return_value; +} +/*[clinic end generated code: output=ee8b1933e4bf4dc4 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index fe84e810760b40..2c6fc4022e3809 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -35,6 +35,7 @@ Iterator, Sequence, ) +from operator import attrgetter from types import FunctionType, NoneType from typing import ( TYPE_CHECKING, @@ -922,41 +923,38 @@ def deprecate_positional_use( params: dict[int, Parameter], ) -> str: assert len(params) > 0 - names = [repr(p.name) for p in params.values()] - first_pos, first_param = next(iter(params.items())) - last_pos, last_param = next(reversed(params.items())) - - # Pretty-print list of names. - pstr = pprint_words(names) - - # For now, assume there's only one deprecation level. - assert first_param.deprecated_positional == last_param.deprecated_positional - thenceforth = first_param.deprecated_positional - assert thenceforth is not None - major, minor = thenceforth + first_pos = next(iter(params)) + last_pos = next(reversed(params)) # Format the deprecation message. - if first_pos == 0: - preamble = "Passing positional arguments to " if len(params) == 1: condition = f"nargs == {first_pos+1}" - if first_pos: - preamble = f"Passing {first_pos+1} positional arguments to " - message = preamble + ( - f"{func.fulldisplayname}() is deprecated. Parameter {pstr} will " - f"become a keyword-only parameter in Python {major}.{minor}." - ) + amount = f"{first_pos+1} " if first_pos else "" + pl = "s" else: condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" - if first_pos: - preamble = ( - f"Passing more than {first_pos} positional " - f"argument{'s' if first_pos != 1 else ''} to " + amount = f"more than {first_pos} " if first_pos else "" + pl = "s" if first_pos != 1 else "" + message = ( + f"Passing {amount}positional argument{pl} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_positional") + ): + names = [repr(p.name) for p in group] + pstr = pprint_words(names) + if len(names) == 1: + message += ( + f" Parameter {pstr} will become a keyword-only parameter " + f"in Python {major}.{minor}." + ) + else: + message += ( + f" Parameters {pstr} will become keyword-only parameters " + f"in Python {major}.{minor}." ) - message = preamble + ( - f"{func.fulldisplayname}() is deprecated. Parameters {pstr} will " - f"become keyword-only parameters in Python {major}.{minor}." - ) # Append deprecation warning to docstring. docstring = textwrap.fill(f"Note: {message}") @@ -977,19 +975,8 @@ def deprecate_keyword_use( argname_fmt: str | None, ) -> str: assert len(params) > 0 - names = [repr(p.name) for p in params.values()] - first_param = next(iter(params.values())) last_param = next(reversed(params.values())) - # Pretty-print list of names. - pstr = pprint_words(names) - - # For now, assume there's only one deprecation level. - assert first_param.deprecated_keyword == last_param.deprecated_keyword - thenceforth = first_param.deprecated_keyword - assert thenceforth is not None - major, minor = thenceforth - # Format the deprecation message. containscheck = "" conditions = [] @@ -1013,16 +1000,25 @@ def deprecate_keyword_use( condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" else: condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" - if len(params) == 1: - what1 = "argument" - what2 = "parameter" - else: - what1 = "arguments" - what2 = "parameters" + names = [repr(p.name) for p in params.values()] + pstr = pprint_words(names) + pl = 's' if len(params) != 1 else '' message = ( - f"Passing keyword {what1} {pstr} to {func.fulldisplayname}() is deprecated. " - f"Corresponding {what2} will become positional-only in Python {major}.{minor}." + f"Passing keyword argument{pl} {pstr} to " + f"{func.fulldisplayname}() is deprecated." ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_keyword") + ): + names = [repr(p.name) for p in group] + pstr = pprint_words(names) + pl = 's' if len(names) != 1 else '' + message += ( + f" Parameter{pl} {pstr} will become positional-only " + f"in Python {major}.{minor}." + ) + if containscheck: errcheck = f""" if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail @@ -5528,9 +5524,15 @@ def parse_star(self, function: Function, version: VersionTuple | None) -> None: self.keyword_only = True else: if self.keyword_only: - fail(f"Function {function.name!r}: '* [from ...]' must come before '*'") + fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") if self.deprecated_positional: - fail(f"Function {function.name!r} uses '* [from ...]' more than once.") + if self.deprecated_positional == version: + fail(f"Function {function.name!r} uses '* [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_positional < version: + fail(f"Function {function.name!r}: '* [from " + f"{version[0]}.{version[1]}]' must precede '* [from " + f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") self.deprecated_positional = version def parse_opening_square_bracket(self, function: Function) -> None: @@ -5582,7 +5584,13 @@ def parse_slash(self, function: Function, version: VersionTuple | None) -> None: fail(f"Function {function.name!r} uses '/' more than once.") else: if self.deprecated_keyword: - fail(f"Function {function.name!r} uses '/ [from ...]' more than once.") + if self.deprecated_keyword == version: + fail(f"Function {function.name!r} uses '/ [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_keyword > version: + fail(f"Function {function.name!r}: '/ [from " + f"{version[0]}.{version[1]}]' must precede '/ [from " + f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") if self.deprecated_positional: fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") if self.keyword_only: @@ -5613,7 +5621,7 @@ def parse_slash(self, function: Function, version: VersionTuple | None) -> None: if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: if version is None: p.kind = inspect.Parameter.POSITIONAL_ONLY - else: + elif p.deprecated_keyword is None: p.deprecated_keyword = version def state_parameter_docstring_start(self, line: str) -> None: From 47022a079eb9d2a2af781abae3de4a71f80247c2 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 21 Aug 2023 10:41:34 -0500 Subject: [PATCH 226/598] docs: fix grammar in isolating-extensions.rst (#108037) --- Doc/howto/isolating-extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 2551fbe87b5c2a..8f3787f2d2f145 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -64,7 +64,7 @@ Enter Per-Module State Instead of focusing on per-interpreter state, Python's C API is evolving to better support the more granular *per-module* state. -This means that C-level data is be attached to a *module object*. +This means that C-level data should be attached to a *module object*. Each interpreter creates its own module object, keeping the data separate. For testing the isolation, multiple module objects corresponding to a single extension can even be loaded in a single interpreter. From 10a91d7e98d847b05292eab828ff9ae51308d3ee Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:31:30 +0100 Subject: [PATCH 227/598] gh-108113: Make it possible to create an optimized AST (#108154) --- Doc/library/ast.rst | 7 +++- Doc/whatsnew/3.13.rst | 14 +++++++ Include/cpython/compile.h | 3 +- Lib/ast.py | 6 ++- Lib/test/test_ast.py | 28 +++++++++++++ Lib/test/test_builtin.py | 41 +++++++++++++++---- ...-08-18-18-21-27.gh-issue-108113.1h0poE.rst | 8 ++++ Parser/asdl_c.py | 3 ++ Python/Python-ast.c | 3 ++ Python/pythonrun.c | 25 +++++++++++ 10 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index cd657aedf6d23d..2237a07eb9d8aa 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2122,10 +2122,12 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. + filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` + and ``ast.PyCF_OPTIMIZED_AST`` otherwise. If ``type_comments=True`` is given, the parser is modified to check and return type comments as specified by :pep:`484` and :pep:`526`. @@ -2171,6 +2173,7 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.13 The minimum supported version for feature_version is now (3,7) + The ``optimize`` argument was added. .. function:: unparse(ast_obj) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 47b868bad31923..bfab868d1c5b62 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -85,6 +85,12 @@ Other Language Changes This change will affect tools using docstrings, like :mod:`doctest`. (Contributed by Inada Naoki in :gh:`81283`.) +* The :func:`compile` built-in can now accept a new flag, + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` + except that the returned ``AST`` is optimized according to the value + of the ``optimize`` argument. + (Contributed by Irit Katriel in :gh:`108113`). + New Modules =========== @@ -94,6 +100,14 @@ New Modules Improved Modules ================ +ast +--- + +* :func:`ast.parse` now accepts an optional argument ``optimize`` + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized ``AST``. + (Contributed by Irit Katriel in :gh:`108113`). + array ----- diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index e6cd39af2ba739..ae17cef554fa17 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -19,9 +19,10 @@ #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 #define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000 +#define PyCF_OPTIMIZED_AST (0x8000 | PyCF_ONLY_AST) #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \ - PyCF_ALLOW_INCOMPLETE_INPUT) + PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_OPTIMIZED_AST) typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ diff --git a/Lib/ast.py b/Lib/ast.py index a307f3ecd06175..45b95963f81885 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -32,13 +32,15 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None): + type_comments=False, feature_version=None, optimize=-1): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). Pass type_comments=True to get back type comments where the syntax allows. """ flags = PyCF_ONLY_AST + if optimize > 0: + flags |= PyCF_OPTIMIZED_AST if type_comments: flags |= PyCF_TYPE_COMMENTS if feature_version is None: @@ -50,7 +52,7 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version) + _feature_version=feature_version, optimize=optimize) def literal_eval(node_or_string): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 5346b39043f0f5..f3c7229f0b6c76 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -357,6 +357,34 @@ def test_ast_validation(self): tree = ast.parse(snippet) compile(tree, '', 'exec') + def test_optimization_levels__debug__(self): + cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] + for (optval, expected) in cases: + with self.subTest(optval=optval, expected=expected): + res = ast.parse("__debug__", optimize=optval) + self.assertIsInstance(res.body[0], ast.Expr) + if isinstance(expected, bool): + self.assertIsInstance(res.body[0].value, ast.Constant) + self.assertEqual(res.body[0].value.value, expected) + else: + self.assertIsInstance(res.body[0].value, ast.Name) + self.assertEqual(res.body[0].value.id, expected) + + def test_optimization_levels_const_folding(self): + folded = ('Expr', (1, 0, 1, 5), ('Constant', (1, 0, 1, 5), 3, None)) + not_folded = ('Expr', (1, 0, 1, 5), + ('BinOp', (1, 0, 1, 5), + ('Constant', (1, 0, 1, 1), 1, None), + ('Add',), + ('Constant', (1, 4, 1, 5), 2, None))) + + cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] + for (optval, expected) in cases: + with self.subTest(optval=optval): + tree = ast.parse("1 + 2", optimize=optval) + res = to_tuple(tree.body[0]) + self.assertEqual(res, expected) + def test_invalid_position_information(self): invalid_linenos = [ (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f5a5c037f1bf1b..ee3ba6ab07bbdf 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -369,16 +369,17 @@ def f(): """doc""" (1, False, 'doc', False, False), (2, False, None, False, False)] for optval, *expected in values: + with self.subTest(optval=optval): # test both direct compilation and compilation via AST - codeobjs = [] - codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) - codeobjs.append(compile(tree, "", "exec", optimize=optval)) - for code in codeobjs: - ns = {} - exec(code, ns) - rv = ns['f']() - self.assertEqual(rv, tuple(expected)) + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, tuple(expected)) def test_compile_top_level_await_no_coro(self): """Make sure top level non-await codes get the correct coroutine flags""" @@ -517,6 +518,28 @@ def test_compile_async_generator(self): exec(co, glob) self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) + def test_compile_ast(self): + args = ("a*(1+2)", "f.py", "exec") + raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] + opt = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + + for tree in (raw, opt): + self.assertIsInstance(tree.value, ast.BinOp) + self.assertIsInstance(tree.value.op, ast.Mult) + self.assertIsInstance(tree.value.left, ast.Name) + self.assertEqual(tree.value.left.id, 'a') + + raw_right = raw.value.right # expect BinOp(1, '+', 2) + self.assertIsInstance(raw_right, ast.BinOp) + self.assertIsInstance(raw_right.left, ast.Constant) + self.assertEqual(raw_right.left.value, 1) + self.assertIsInstance(raw_right.right, ast.Constant) + self.assertEqual(raw_right.right.value, 2) + + opt_right = opt.value.right # expect Constant(3) + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, 3) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst new file mode 100644 index 00000000000000..66680578c9b43b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst @@ -0,0 +1,8 @@ +The :func:`compile` built-in can now accept a new flag, +``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` +except that the returned ``AST`` is optimized according to the value +of the ``optimize`` argument. + +:func:`ast.parse` now accepts an optional argument ``optimize`` +which is passed on to the :func:`compile` built-in. This makes it +possible to obtain an optimized ``AST``. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 2a36610527f898..1733cd4b15aa4c 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1208,6 +1208,9 @@ def visitModule(self, mod): self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) {', 1) self.emit("return -1;", 2) self.emit('}', 1) + self.emit('if (PyModule_AddIntMacro(m, PyCF_OPTIMIZED_AST) < 0) {', 1) + self.emit("return -1;", 2) + self.emit('}', 1) for dfn in mod.dfns: self.visit(dfn) self.emit("return 0;", 1) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 8047b1259c5d86..60dd121d60b8d0 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -12659,6 +12659,9 @@ astmodule_exec(PyObject *m) if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) { return -1; } + if (PyModule_AddIntMacro(m, PyCF_OPTIMIZED_AST) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "mod", state->mod_type) < 0) { return -1; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 721c527745c44a..b2e04cfa317c00 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -21,6 +21,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_symtable.h" // _PyFuture_FromAST() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() @@ -1790,6 +1791,24 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, return NULL; } +static int +ast_optimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, + int optimize, PyArena *arena) +{ + PyFutureFeatures future; + if (!_PyFuture_FromAST(mod, filename, &future)) { + return -1; + } + int flags = future.ff_features | cf->cf_flags; + if (optimize == -1) { + optimize = _Py_GetConfig()->optimization_level; + } + if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + return -1; + } + return 0; +} + PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) @@ -1806,6 +1825,12 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { + if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { + if (ast_optimize(mod, filename, flags, optimize, arena) < 0) { + _PyArena_Free(arena); + return NULL; + } + } PyObject *result = PyAST_mod2obj(mod); _PyArena_Free(arena); return result; From db55383829ccd5ce80c551d60f26851346741fdf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Aug 2023 20:05:15 +0300 Subject: [PATCH 228/598] gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) --- Doc/whatsnew/3.11.rst | 26 ++++++++-------- Doc/whatsnew/3.3.rst | 66 ++++++++++++++++++++-------------------- Doc/whatsnew/3.6.rst | 6 ++-- Doc/whatsnew/3.9.rst | 4 +-- Misc/NEWS.d/3.11.0b1.rst | 6 ++-- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index ec5263ec35c52f..fce00d37555c2e 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2591,22 +2591,22 @@ Pending Removal in Python 3.12 The following C APIs have been deprecated in earlier Python releases, and will be removed in Python 3.12. -* :c:func:`PyUnicode_AS_DATA` -* :c:func:`PyUnicode_AS_UNICODE` -* :c:func:`PyUnicode_AsUnicodeAndSize` -* :c:func:`PyUnicode_AsUnicode` -* :c:func:`PyUnicode_FromUnicode` -* :c:func:`PyUnicode_GET_DATA_SIZE` -* :c:func:`PyUnicode_GET_SIZE` -* :c:func:`PyUnicode_GetSize` +* :c:func:`!PyUnicode_AS_DATA` +* :c:func:`!PyUnicode_AS_UNICODE` +* :c:func:`!PyUnicode_AsUnicodeAndSize` +* :c:func:`!PyUnicode_AsUnicode` +* :c:func:`!PyUnicode_FromUnicode` +* :c:func:`!PyUnicode_GET_DATA_SIZE` +* :c:func:`!PyUnicode_GET_SIZE` +* :c:func:`!PyUnicode_GetSize` * :c:func:`PyUnicode_IS_COMPACT` * :c:func:`PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` -* :c:func:`Py_UNICODE_WSTR_LENGTH` -* :c:func:`_PyUnicode_AsUnicode` -* :c:macro:`PyUnicode_WCHAR_KIND` +* :c:func:`!PyUnicode_WSTR_LENGTH` +* :c:func:`!_PyUnicode_AsUnicode` +* :c:macro:`!PyUnicode_WCHAR_KIND` * :c:type:`PyUnicodeObject` -* :c:func:`PyUnicode_InternImmortal()` +* :c:func:`!PyUnicode_InternImmortal` .. _whatsnew311-c-api-removed: @@ -2614,7 +2614,7 @@ and will be removed in Python 3.12. Removed ------- -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been +* :c:func:`!PyFrame_BlockSetup` and :c:func:`!PyFrame_BlockPop` have been removed. (Contributed by Mark Shannon in :issue:`40222`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index bcdc0222be6dea..7e12c279065675 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -249,7 +249,7 @@ Changes introduced by :pep:`393` are the following: non-BMP code points. * The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF`` - in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns + in hexadecimal). The :c:func:`!PyUnicode_GetMax` function still returns either ``0xFFFF`` or ``0x10FFFF`` for backward compatibility, and it should not be used with the new Unicode API (see :issue:`13054`). @@ -2196,7 +2196,7 @@ Changes to Python's build process and to the C API include: * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: - :c:data:`PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, + :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` * :c:macro:`PyUnicode_MAX_CHAR_VALUE` @@ -2270,58 +2270,58 @@ removed in Python 4. All functions using this type are deprecated: Unicode functions and methods using :c:type:`Py_UNICODE` and :c:expr:`Py_UNICODE*` types: -* :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or +* :c:macro:`!PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_FromKindAndData` -* :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, - :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` -* :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with +* :c:macro:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_AsUnicode`, + :c:func:`!PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` +* :c:macro:`!PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` -* :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use +* :c:macro:`!PyUnicode_GET_SIZE`, :c:func:`!PyUnicode_GetSize`: use :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` -* :c:macro:`PyUnicode_GET_DATA_SIZE`: use +* :c:macro:`!PyUnicode_GET_DATA_SIZE`: use ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready strings) -* :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or +* :c:func:`!PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or :c:func:`PyUnicode_AsWideCharString` -* :c:func:`PyUnicode_GetMax` +* :c:func:`!PyUnicode_GetMax` Functions and macros manipulating Py_UNICODE* strings: -* :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or +* :c:macro:`!Py_UNICODE_strlen()`: use :c:func:`PyUnicode_GetLength` or :c:macro:`PyUnicode_GET_LENGTH` -* :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcat()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_FromFormat` -* :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, - :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcpy()`, :c:macro:`!Py_UNICODE_strncpy()`, + :c:macro:`!Py_UNICODE_COPY()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` -* :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` -* :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` -* :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use +* :c:macro:`!Py_UNICODE_strcmp()`: use :c:func:`PyUnicode_Compare` +* :c:macro:`!Py_UNICODE_strncmp()`: use :c:func:`PyUnicode_Tailmatch` +* :c:macro:`!Py_UNICODE_strchr()`, :c:macro:`!Py_UNICODE_strrchr()`: use :c:func:`PyUnicode_FindChar` -* :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` -* :c:macro:`Py_UNICODE_MATCH` +* :c:macro:`!Py_UNICODE_FILL()`: use :c:func:`PyUnicode_Fill` +* :c:macro:`!Py_UNICODE_MATCH` Encoders: -* :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` -* :c:func:`PyUnicode_EncodeUTF7` -* :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or +* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_EncodeUTF7` +* :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` -* :c:func:`PyUnicode_EncodeUTF32` -* :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeUTF32` +* :c:func:`!PyUnicode_EncodeUTF16` +* :c:func:`!PyUnicode_EncodeUnicodeEscape` use :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeRawUnicodeEscape` use :c:func:`PyUnicode_AsRawUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` -* :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` -* :c:func:`PyUnicode_EncodeCharmap` -* :c:func:`PyUnicode_TranslateCharmap` -* :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or +* :c:func:`!PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` +* :c:func:`!PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` +* :c:func:`!PyUnicode_EncodeCharmap` +* :c:func:`!PyUnicode_TranslateCharmap` +* :c:func:`!PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) -* :c:func:`PyUnicode_EncodeDecimal`, - :c:func:`PyUnicode_TransformDecimalToASCII` +* :c:func:`!PyUnicode_EncodeDecimal`, + :c:func:`!PyUnicode_TransformDecimalToASCII` Deprecated features diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 4359a9012dd53c..c15d8be651fd17 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2066,9 +2066,9 @@ environment. (Contributed by Brett Cannon in :issue:`25154`.) Deprecated functions and types of the C API ------------------------------------------- -Undocumented functions :c:func:`PyUnicode_AsEncodedObject`, -:c:func:`PyUnicode_AsDecodedObject`, :c:func:`PyUnicode_AsEncodedUnicode` -and :c:func:`PyUnicode_AsDecodedUnicode` are deprecated now. +Undocumented functions :c:func:`!PyUnicode_AsEncodedObject`, +:c:func:`!PyUnicode_AsDecodedObject`, :c:func:`!PyUnicode_AsEncodedUnicode` +and :c:func:`!PyUnicode_AsDecodedUnicode` are deprecated now. Use the :ref:`generic codec based API ` instead. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 3e3e0ff5c41f4f..5891d9a55534fd 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1370,8 +1370,8 @@ Porting to Python 3.9 (Contributed by Victor Stinner in :issue:`40241`.) * The ``Py_UNICODE_COPY``, ``Py_UNICODE_FILL``, ``PyUnicode_WSTR_LENGTH``, - :c:func:`PyUnicode_FromUnicode`, :c:func:`PyUnicode_AsUnicode`, - ``_PyUnicode_AsUnicode``, and :c:func:`PyUnicode_AsUnicodeAndSize` are + :c:func:`!PyUnicode_FromUnicode`, :c:func:`!PyUnicode_AsUnicode`, + ``_PyUnicode_AsUnicode``, and :c:func:`!PyUnicode_AsUnicodeAndSize` are marked as deprecated in C. They have been deprecated by :pep:`393` since Python 3.3. (Contributed by Inada Naoki in :issue:`36346`.) diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 71efc21cbc4a61..766ada4f8e1176 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -2068,9 +2068,9 @@ casts when the Python C API is used in C++. Patch by Victor Stinner. .. nonce: Cx-95G .. section: C API -Mark functions as deprecated by :pep:`623`: :c:func:`PyUnicode_AS_DATA`, -:c:func:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_GET_DATA_SIZE`, -:c:func:`PyUnicode_GET_SIZE`. Patch by Victor Stinner. +Mark functions as deprecated by :pep:`623`: :c:func:`!PyUnicode_AS_DATA`, +:c:func:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_GET_DATA_SIZE`, +:c:func:`!PyUnicode_GET_SIZE`. Patch by Victor Stinner. .. From 21c0844742cf15db8e56e8848ecbb2e25f314aed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Aug 2023 19:15:52 +0200 Subject: [PATCH 229/598] gh-108220: Internal header files require Py_BUILD_CORE to be defined (#108221) * pycore_intrinsics.h does nothing if included twice (add #ifndef and #define). * Update Tools/cases_generator/generate_cases.py to generate the Py_BUILD_CORE test. * _bz2, _lzma, _opcode and zlib extensions now define the Py_BUILD_CORE_MODULE macro to use internal headers (pycore_code.h, pycore_intrinsics.h and pycore_blocks_output_buffer.h). --- Include/internal/pycore_blocks_output_buffer.h | 6 +++++- Include/internal/pycore_code.h | 4 ++++ Include/internal/pycore_codecs.h | 4 ++++ Include/internal/pycore_emscripten_signal.h | 4 ++++ Include/internal/pycore_fileutils.h | 2 +- Include/internal/pycore_fileutils_windows.h | 2 +- Include/internal/pycore_frame.h | 4 ++++ Include/internal/pycore_import.h | 4 ++++ Include/internal/pycore_instruments.h | 4 +++- Include/internal/pycore_intrinsics.h | 8 ++++++++ Include/internal/pycore_opcode_metadata.h | 4 ++++ Modules/_bz2module.c | 4 ++++ Modules/_lzmamodule.c | 4 ++++ Modules/_opcode.c | 4 ++++ Modules/zlibmodule.c | 4 ++++ Tools/cases_generator/generate_cases.py | 7 +++++++ 16 files changed, 65 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 28cf6fba4eeba2..573e10359b7bd2 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -40,6 +40,10 @@ extern "C" { #include "Python.h" +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + typedef struct { // List of bytes objects PyObject *list; @@ -314,4 +318,4 @@ _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) #ifdef __cplusplus } #endif -#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ \ No newline at end of file +#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 00099376635e9b..f5127a81144353 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #define CODE_MAX_WATCHERS 8 /* PEP 659 diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2465192eacd5e..a2a7151d50ade7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + extern PyObject* _PyCodec_Lookup(const char *encoding); /* Text codec specific encoding and decoding API. diff --git a/Include/internal/pycore_emscripten_signal.h b/Include/internal/pycore_emscripten_signal.h index 8b3287d85da4b2..d1bcb9a92c7726 100644 --- a/Include/internal/pycore_emscripten_signal.h +++ b/Include/internal/pycore_emscripten_signal.h @@ -3,6 +3,10 @@ #if defined(__EMSCRIPTEN__) +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + void _Py_CheckEmscriptenSignals(void); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 0ed139f79b1429..25b383b9780adc 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -5,7 +5,7 @@ extern "C" { #endif #ifndef Py_BUILD_CORE -# error "Py_BUILD_CORE must be defined to include this header" +# error "this header requires Py_BUILD_CORE define" #endif #include /* struct lconv */ diff --git a/Include/internal/pycore_fileutils_windows.h b/Include/internal/pycore_fileutils_windows.h index e804d385e76708..b79aa9fb465376 100644 --- a/Include/internal/pycore_fileutils_windows.h +++ b/Include/internal/pycore_fileutils_windows.h @@ -5,7 +5,7 @@ extern "C" { #endif #ifndef Py_BUILD_CORE -# error "Py_BUILD_CORE must be defined to include this header" +# error "this header requires Py_BUILD_CORE define" #endif #ifdef MS_WINDOWS diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 0dc2a1814cb1d5..ae77367f6a3c9b 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include #include // offsetof() #include "pycore_code.h" // STATS diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 077508e6c58f6c..34f572bd592969 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -5,6 +5,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_time.h" // _PyTime_t diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 56de9f87171484..e15447acec2c0c 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -1,7 +1,9 @@ - #ifndef Py_INTERNAL_INSTRUMENT_H #define Py_INTERNAL_INSTRUMENT_H +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif #include "pycore_bitutils.h" // _Py_popcount32 #include "pycore_frame.h" diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 37d4efc12bb771..3a8dd95cff8e5d 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,3 +1,9 @@ +#ifndef Py_INTERNAL_INTRINSIC_H +#define Py_INTERNAL_INTRINSIC_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif /* Unary Functions: */ #define INTRINSIC_1_INVALID 0 @@ -40,3 +46,5 @@ typedef struct { extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; + +#endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 396d194ed2734e..fab91e611b6175 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -3,6 +3,10 @@ // Python/bytecodes.c // Do not edit! +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 0a84f25ca4cbe7..3d0d4ee5e79c2b 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -1,5 +1,9 @@ /* _bz2 - Low-level Python interface to libbzip2. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index c548f8fa3839e0..eb90c308d16d19 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -5,6 +5,10 @@ */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 4f85e7ee26de2a..3e13dbb6edc26e 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -1,3 +1,7 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "compile.h" #include "opcode.h" diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a98a37adadcff0..9b76afa0e56f76 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -3,6 +3,10 @@ /* Windows users: read Python's PCbuild\readme.txt */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "zlib.h" diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 7b1880b98a8feb..c3b729df44a6a2 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -9,6 +9,7 @@ import os import posixpath import sys +import textwrap import typing from collections.abc import Iterator @@ -403,6 +404,12 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_provenance_header() + self.out.emit("\n" + textwrap.dedent(""" + #ifndef Py_BUILD_CORE + # error "this header requires Py_BUILD_CORE define" + #endif + """).strip()) + self.out.emit("\n#include ") self.write_pseudo_instrs() From b16ecb88e70d696a93ce993661973330baeafee1 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 21 Aug 2023 13:50:35 -0400 Subject: [PATCH 230/598] gh-108223: Add --disable-gil to configure (gh-108227) The `--disable-gil` flags does not do anything yet other than define the Py_NOGIL macro. This is intended to support setting up additional buildbots. --- configure | 31 +++++++++++++++++++++++++++++++ configure.ac | 15 +++++++++++++++ pyconfig.h.in | 3 +++ 3 files changed, 49 insertions(+) diff --git a/configure b/configure index aaacf8d2669c16..f78b45a2c72dd2 100755 --- a/configure +++ b/configure @@ -1103,6 +1103,7 @@ with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes enable_test_modules +enable_gil ' ac_precious_vars='build_alias host_alias @@ -1803,6 +1804,8 @@ Optional Features: use big digits (30 or 15 bits) for Python longs (default is 30)] --disable-test-modules don't build nor install test modules + --disable-gil enable experimental support for running without the + GIL (default is no) Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] @@ -27904,6 +27907,34 @@ fi printf "%s\n" "$TEST_MODULES" >&6; } +# Check for --disable-gil +# --disable-gil +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-gil" >&5 +printf %s "checking for --disable-gil... " >&6; } +# Check whether --enable-gil was given. +if test ${enable_gil+y} +then : + enableval=$enable_gil; if test "x$enable_gil" = xyes +then : + disable_gil=no +else $as_nop + disable_gil=yes +fi +else $as_nop + disable_gil=no + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_gil" >&5 +printf "%s\n" "$disable_gil" >&6; } + +if test "$disable_gil" = "yes" +then + +printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h + +fi + # stdlib not available diff --git a/configure.ac b/configure.ac index ddf6da0b9da123..fd18a452f2b610 100644 --- a/configure.ac +++ b/configure.ac @@ -7126,6 +7126,21 @@ AC_ARG_ENABLE([test-modules], AC_MSG_RESULT([$TEST_MODULES]) AC_SUBST([TEST_MODULES]) +# Check for --disable-gil +# --disable-gil +AC_MSG_CHECKING([for --disable-gil]) +AC_ARG_ENABLE([gil], + [AS_HELP_STRING([--disable-gil], [enable experimental support for running without the GIL (default is no)])], + [AS_VAR_IF([enable_gil], [yes], [disable_gil=no], [disable_gil=yes])], [disable_gil=no] +) +AC_MSG_RESULT([$disable_gil]) + +if test "$disable_gil" = "yes" +then + AC_DEFINE([Py_NOGIL], [1], + [Define if you want to disable the GIL]) +fi + AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ AS_VAR_SET([py_cv_module_]mod, [n/a])]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 181dc3d7d11370..418ccade8e8bb1 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1609,6 +1609,9 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM +/* Define if you want to disable the GIL */ +#undef Py_NOGIL + /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS From 0dd3fc2a640b273979f94299b545e1e40ac0633c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Aug 2023 20:05:59 +0200 Subject: [PATCH 231/598] gh-108216: Cleanup #include in internal header files (#108228) * Add missing includes. * Remove unused includes. * Update old include/symbol names to newer names. * Mention at least one included symbol. * Sort includes. * Update Tools/cases_generator/generate_cases.py used to generated pycore_opcode_metadata.h. * Update Parser/asdl_c.py used to generate pycore_ast.h. * Cleanup also includes in _testcapimodule.c and _testinternalcapi.c. --- Include/internal/pycore_ast.h | 2 +- Include/internal/pycore_atomic.h | 14 +++---- Include/internal/pycore_bitutils.h | 4 +- Include/internal/pycore_ceval.h | 7 ++-- Include/internal/pycore_ceval_state.h | 2 - Include/internal/pycore_condvar.h | 6 +-- Include/internal/pycore_context.h | 2 +- Include/internal/pycore_dict.h | 4 +- Include/internal/pycore_faulthandler.h | 2 +- Include/internal/pycore_fileutils.h | 5 ++- Include/internal/pycore_gil.h | 4 +- Include/internal/pycore_global_objects.h | 6 +-- Include/internal/pycore_import.h | 2 +- Include/internal/pycore_instruments.h | 4 +- Include/internal/pycore_interp.h | 12 +++--- Include/internal/pycore_object.h | 1 - Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_opcode_utils.h | 2 +- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_pymem_init.h | 2 - Include/internal/pycore_pystate.h | 2 +- Include/internal/pycore_pythread.h | 10 ++--- Include/internal/pycore_runtime.h | 13 +++---- Include/internal/pycore_runtime_init.h | 16 +++++--- .../internal/pycore_runtime_init_generated.h | 3 ++ Include/internal/pycore_signal.h | 4 +- Include/internal/pycore_typeobject.h | 4 +- Modules/_json.c | 2 +- Modules/_testcapimodule.c | 6 +-- Modules/_testinternalcapi.c | 38 +++++++++---------- Parser/asdl_c.py | 2 +- Tools/cases_generator/generate_cases.py | 2 +- 32 files changed, 92 insertions(+), 95 deletions(-) diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index b568902bb1e381..f222d485e0b54b 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -10,7 +10,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_asdl.h" +#include "pycore_asdl.h" // _ASDL_SEQ_HEAD typedef struct _mod *mod_ty; diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h index 425d69f868b52b..48d246ea08f3d9 100644 --- a/Include/internal/pycore_atomic.h +++ b/Include/internal/pycore_atomic.h @@ -8,19 +8,19 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "dynamic_annotations.h" /* _Py_ANNOTATE_MEMORY_ORDER */ -#include "pyconfig.h" +#include "pyconfig.h" // HAVE_STD_ATOMIC +#include "dynamic_annotations.h" // _Py_ANNOTATE_MEMORY_ORDER #ifdef HAVE_STD_ATOMIC -# include +# include // atomic_store_explicit() #endif #if defined(_MSC_VER) -#include -#if defined(_M_IX86) || defined(_M_X64) -# include -#endif +# include // _InterlockedExchange64() +# if defined(_M_IX86) || defined(_M_X64) +# include // _InterlockedExchange_HLEAcquire() +# endif #endif /* This is modeled after the atomics interface from C1x, according to diff --git a/Include/internal/pycore_bitutils.h b/Include/internal/pycore_bitutils.h index e6bf61ef425bd8..50f69377523818 100644 --- a/Include/internal/pycore_bitutils.h +++ b/Include/internal/pycore_bitutils.h @@ -26,10 +26,10 @@ extern "C" { #endif #ifdef _MSC_VER - /* Get _byteswap_ushort(), _byteswap_ulong(), _byteswap_uint64() */ -# include +# include // _byteswap_uint64() #endif + static inline uint16_t _Py_bswap16(uint16_t word) { diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 0e3a99be8c36aa..f32ed3b7e30b7e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_interp.h" // PyInterpreterState.eval_frame +#include "pycore_pystate.h" // _PyThreadState_GET() + /* Forward declarations */ struct pyruntimestate; struct _ceval_runtime_state; @@ -16,10 +19,6 @@ struct _ceval_runtime_state; # define Py_DEFAULT_RECURSION_LIMIT 1000 #endif -#include "pycore_interp.h" // PyInterpreterState.eval_frame -#include "pycore_pystate.h" // _PyThreadState_GET() - - extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 1ebfcc9bebd0ab..6e3d669dc646af 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_gil.h" // struct _gil_runtime_state diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 981c962bf7dfdf..db8682a4c077aa 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -10,7 +10,7 @@ not present in unistd.h. But they still can be implemented as an external library (e.g. gnu pth in pthread emulation) */ # ifdef HAVE_PTHREAD_H -# include /* _POSIX_THREADS */ +# include // _POSIX_THREADS # endif #endif @@ -21,7 +21,7 @@ #define Py_HAVE_CONDVAR #ifdef HAVE_PTHREAD_H -# include +# include // pthread_mutex_t #endif #define PyMUTEX_T pthread_mutex_t @@ -38,7 +38,7 @@ /* include windows if it hasn't been done before */ #define WIN32_LEAN_AND_MEAN -#include +#include // CRITICAL_SECTION /* options */ /* non-emulated condition variables are provided for those that want diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 52dfe3ef233874..f1898cf873312b 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,7 +5,7 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_hamt.h" /* PyHamtObject */ +#include "pycore_hamt.h" // PyHamtObject extern PyTypeObject _PyContextTokenMissing_Type; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 2ad6ef0f7c04d5..df7bc7e58f6c97 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,9 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_dict_state.h" -#include "pycore_object.h" -#include "pycore_runtime.h" // _PyRuntime +#include "pycore_object.h" // PyDictOrValues // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index e6aec7745a6479..6dd7d8d7ca9792 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef HAVE_SIGACTION -# include +# include // sigaction #endif diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 25b383b9780adc..96e3d1c7abcdde 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -8,7 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include /* struct lconv */ +#include // struct lconv + /* A routine to check if a file descriptor can be select()-ed. */ #ifdef _MSC_VER @@ -268,7 +269,7 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t // so provide our own implementations. Remove them in case they get added // to the Games API family #if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) -#include +#include // HRESULT extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 8ebad37b686cd4..a1a15070eef461 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -8,8 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_condvar.h" /* PyCOND_T */ +#include "pycore_atomic.h" // _Py_atomic_address +#include "pycore_condvar.h" // PyCOND_T #ifndef Py_HAVE_CONDVAR # error You need either a POSIX-compatible or a Windows system! diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 442f8516278b02..327fcc24cb29f1 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -8,11 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_gc.h" // PyGC_Head +#include "pycore_context.h" // _PyContextTokenMissing +#include "pycore_gc.h" // _PyGC_Head_UNUSED #include "pycore_global_strings.h" // struct _Py_global_strings #include "pycore_hamt.h" // PyHamtNode_Bitmap -#include "pycore_context.h" // _PyContextTokenMissing +#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_typeobject.h" // pytype_slotdef diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 34f572bd592969..e23f9931cea734 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -103,7 +103,7 @@ struct _import_state { }; #ifdef HAVE_DLOPEN -# include +# include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW # define _Py_DLOPEN_FLAGS RTLD_NOW # else diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e15447acec2c0c..43214aef7f7a1b 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -6,9 +6,7 @@ #endif #include "pycore_bitutils.h" // _Py_popcount32 -#include "pycore_frame.h" - -#include "cpython/code.h" +#include "pycore_frame.h" // _PyInterpreterFrame #ifdef __cplusplus extern "C" { diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 91c473e58eaba2..c21c352608d161 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include +#include // bool #include "pycore_ast_state.h" // struct ast_state #include "pycore_atexit.h" // struct atexit_state @@ -21,16 +21,16 @@ extern "C" { #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_function.h" // FUNC_MAX_WATCHERS -#include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_global_objects.h" // struct _Py_interp_static_objects +#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state -#include "pycore_object_state.h" // struct _py_object_state -#include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_object_state.h" // struct _py_object_state +#include "pycore_obmalloc.h" // struct _obmalloc_state #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeobject.h" // struct type_cache +#include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 857d6efec3b3b1..5e2d13a5220556 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,7 +12,6 @@ extern "C" { #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_runtime.h" // _PyRuntime /* Check if an object is consistent. For example, ensure that the reference counter is greater than or equal to 1, and ensure that ob_type is not NULL. diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fab91e611b6175..e35db0c4c5a59d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -7,7 +7,7 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include +#include // bool #define IS_PSEUDO_INSTR(OP) ( \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 50ff2af38d2cbe..f17612908cebdc 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // _PyOpcode_Jump +#include "pycore_opcode.h" // JUMP_FORWARD #define MAX_REAL_OPCODE 254 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2ae657c4e117ff..f9f16c48cbc21c 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_uops.h" +#include "pycore_uops.h" // _PyUOpInstruction int _Py_uop_analyze_and_optimize(PyCodeObject *code, _PyUOpInstruction *trace, int trace_len, int curr_stackentries); diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 78232738cb09d5..119fa16fb911ae 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_pymem.h" - /********************************/ /* the allocators' initializers */ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index acc6cf953343fb..f6ca9102b139e6 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_runtime.h" /* PyRuntimeState */ +#include "pycore_runtime.h" // _PyRuntime /* Check if the current thread is the main thread. diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index f53921494c158f..44846c0fc4b4c3 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -13,9 +13,9 @@ extern "C" { /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include /* _POSIX_THREADS */ -# endif +# ifdef HAVE_PTHREAD_H +# include // _POSIX_THREADS +# endif # ifndef _POSIX_THREADS /* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then enough of the Posix threads package is implemented to support python @@ -34,12 +34,12 @@ extern "C" { #endif /* _POSIX_THREADS */ #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) -# define _USE_PTHREADS +# define _USE_PTHREADS #endif #if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) // monotonic is supported statically. It doesn't mean it works on runtime. -# define CONDATTR_MONOTONIC +# define CONDATTR_MONOTONIC #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 63b485027486bb..4a460803c69bde 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -8,24 +8,23 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atexit.h" // struct atexit_runtime_state -#include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_atexit.h" // struct _atexit_runtime_state +#include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_global_objects.h" // struct _Py_global_objects +#include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState #include "pycore_object_state.h" // struct _py_object_runtime_state #include "pycore_parser.h" // struct _parser_runtime_state -#include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state +#include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_signal.h" // struct _signals_runtime_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state -#include "pycore_typeobject.h" // struct types_runtime_state -#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids +#include "pycore_typeobject.h" // struct _types_runtime_state +#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state struct _getargs_runtime_state { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ea29c69f59acc7..c775a8a7e7eefa 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -8,11 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_long.h" -#include "pycore_object.h" -#include "pycore_parser.h" -#include "pycore_pymem_init.h" -#include "pycore_obmalloc_init.h" +#include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT +#include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_object.h" // _PyObject_HEAD_INIT +#include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT +#include "pycore_parser.h" // _parser_runtime_state_INIT +#include "pycore_pyhash.h" // pyhash_state_INIT +#include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT +#include "pycore_signal.h" // _signals_RUNTIME_INIT +#include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT extern PyTypeObject _PyExc_MemoryError; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 8c0fcdb1bdae1c..694a409fbf4f47 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_long.h" // _PyLong_DIGIT_INIT() + + /* The following is auto-generated by Tools/build/generate_global_objects.py. */ #define _Py_small_ints_INIT { \ _PyLong_DIGIT_INIT(-5), \ diff --git a/Include/internal/pycore_signal.h b/Include/internal/pycore_signal.h index 46b57d556e5ef4..8f876759c0d07f 100644 --- a/Include/internal/pycore_signal.h +++ b/Include/internal/pycore_signal.h @@ -10,8 +10,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" // _Py_atomic_address -#include // NSIG +#include "pycore_atomic.h" // _Py_atomic_address +#include // NSIG // Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index aba672effe3928..4d6a0189c4afee 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -4,12 +4,12 @@ extern "C" { #endif -#include "pycore_moduleobject.h" - #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_moduleobject.h" // PyModuleObject + /* state */ diff --git a/Modules/_json.c b/Modules/_json.c index c7cfe50b52faff..41495e2012f152 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -12,7 +12,7 @@ #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_runtime.h" // _PyRuntime -#include "pycore_global_objects.h" // _Py_ID() +#include "pycore_global_strings.h" // _Py_ID() #include // bool diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 35599f8baa204d..a7a98d1eea5bd1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,8 +18,8 @@ #undef NDEBUG #include "Python.h" -#include "frameobject.h" // PyFrame_New -#include "marshal.h" // PyMarshal_WriteLongToFile +#include "frameobject.h" // PyFrame_New() +#include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX #include @@ -37,7 +37,7 @@ #endif #ifdef bool -# error "The public headers should not include , see bpo-46748" +# error "The public headers should not include , see gh-48924" #endif // Several parts of this module are broken out into files in _testcapi/. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d1082c7dae8aee..5571ae463373e0 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,26 +10,24 @@ #undef NDEBUG #include "Python.h" -#include "pycore_atomic_funcs.h" // _Py_atomic_int_get() -#include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_bytesobject.h" // _PyBytes_Find() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc -#include "pycore_ceval.h" // _PyEval_AddPendingCall -#include "pycore_dict.h" // _PyDictOrValues_GetValues -#include "pycore_fileutils.h" // _Py_normpath -#include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_gc.h" // PyGC_Head -#include "pycore_hashtable.h" // _Py_hashtable_new() -#include "pycore_initconfig.h" // _Py_GetConfigsAsDict() -#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() -#include "pycore_object.h" // _PyObject_IsFreed() -#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() -#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() -#include "pycore_pystate.h" // _PyThreadState_GET() - -#include "frameobject.h" -#include "interpreteridobject.h" // PyInterpreterID_LookUp() -#include "osdefs.h" // MAXPATHLEN +#include "pycore_atomic_funcs.h" // _Py_atomic_int_get() +#include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_bytesobject.h" // _PyBytes_Find() +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc +#include "pycore_ceval.h" // _PyEval_AddPendingCall +#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_fileutils.h" // _Py_normpath +#include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_gc.h" // PyGC_Head +#include "pycore_hashtable.h" // _Py_hashtable_new() +#include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() +#include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() +#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() +#include "pycore_pystate.h" // _PyThreadState_GET() + +#include "interpreteridobject.h" // PyInterpreterID_LookUp() #include "clinic/_testinternalcapi.c.h" diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 1733cd4b15aa4c..bf144871bb7364 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1585,7 +1585,7 @@ def write_header(mod, metadata, f): # error "this header requires Py_BUILD_CORE define" #endif - #include "pycore_asdl.h" + #include "pycore_asdl.h" // _ASDL_SEQ_HEAD """).lstrip()) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index c3b729df44a6a2..6ee68363058116 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -410,7 +410,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No #endif """).strip()) - self.out.emit("\n#include ") + self.out.emit("\n#include // bool") self.write_pseudo_instrs() From 1cc391d9e2ea24ca750005335507b52933fc0b52 Mon Sep 17 00:00:00 2001 From: temach Date: Tue, 22 Aug 2023 00:08:04 +0600 Subject: [PATCH 232/598] gh-108224: Fix asyncio doc inconsistency (#108230) (Spawning subprocesses does not require the event loop to run in the main thread -- only signal handling does.) --- Doc/library/asyncio-dev.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index c7d97008fb490e..a9c3a0183bb72d 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -99,7 +99,7 @@ To schedule a coroutine object from a different OS thread, the # Wait for the result: result = future.result() -To handle signals and to execute subprocesses, the event loop must be +To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a From 18fc543b377f3521b2503e0d013e1d32ccdf9190 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Aug 2023 20:12:22 +0200 Subject: [PATCH 233/598] gh-107211: No longer export pycore_strhex.h functions (#108229) No longer export functions: * _Py_strhex_bytes() * _Py_strhex_with_sep() --- Include/internal/pycore_strhex.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index f427b4d695bd29..225f423912f2c2 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -9,21 +9,24 @@ extern "C" { #endif // Returns a str() containing the hex representation of argbuf. +// Export for '_hashlib' shared extension. PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -PyAPI_FUNC(PyObject*) _Py_strhex_bytes( +extern PyObject* _Py_strhex_bytes( const char* argbuf, const Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -PyAPI_FUNC(PyObject*) _Py_strhex_with_sep( +extern PyObject* _Py_strhex_with_sep( const char* argbuf, const Py_ssize_t arglen, PyObject* sep, const int bytes_per_group); + +// Export for 'binascii' shared extension PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( const char* argbuf, const Py_ssize_t arglen, From d63972e289e05b0d82e59f32f107312a8b3de7b5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Aug 2023 21:15:46 +0300 Subject: [PATCH 234/598] gh-107298: Fix C API datetime documentation (GH-108034) --- Doc/c-api/datetime.rst | 71 +++++++++++++++++++++++++++++++++--------- Doc/tools/.nitignore | 1 - 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 72fc07afbf1f4d..97522da773477e 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,54 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following macros. +.. c:type:: PyDateTime_Date + + This subtype of :c:type:`PyObject` represents a Python date object. + +.. c:type:: PyDateTime_DateTime + + This subtype of :c:type:`PyObject` represents a Python datetime object. + +.. c:type:: PyDateTime_Time + + This subtype of :c:type:`PyObject` represents a Python time object. + +.. c:type:: PyDateTime_Delta + + This subtype of :c:type:`PyObject` represents the difference between two datetime values. + +.. c:var:: PyTypeObject PyDateTime_DateType + + This instance of :c:type:`PyTypeObject` represents the Python date type; + it is the same object as :class:`datetime.date` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DateTimeType + + This instance of :c:type:`PyTypeObject` represents the Python datetime type; + it is the same object as :class:`datetime.datetime` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TimeType + + This instance of :c:type:`PyTypeObject` represents the Python time type; + it is the same object as :class:`datetime.time` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DeltaType + + This instance of :c:type:`PyTypeObject` represents Python type for + the difference between two datetime values; + it is the same object as :class:`datetime.timedelta` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TZInfoType + + This instance of :c:type:`PyTypeObject` represents the Python time zone info type; + it is the same object as :class:`datetime.tzinfo` in the Python layer. + + Macro for access to the UTC singleton: .. c:var:: PyObject* PyDateTime_TimeZone_UTC @@ -28,7 +71,7 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always succeeds. @@ -41,7 +84,7 @@ Type-check macros: .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -54,7 +97,7 @@ Type-check macros: .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -67,7 +110,7 @@ Type-check macros: .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always succeeds. @@ -80,7 +123,7 @@ Type-check macros: .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always succeeds. @@ -133,7 +176,7 @@ Macros to create objects: :class:`datetime.timedelta` objects. -.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) +.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) Return a :class:`datetime.timezone` object with an unnamed fixed offset represented by the *offset* argument. @@ -141,7 +184,7 @@ Macros to create objects: .. versionadded:: 3.7 -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) Return a :class:`datetime.timezone` object with a fixed offset represented by the *offset* argument and with tzname *name*. @@ -150,8 +193,8 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is +:c:type:`PyDateTime_Date`, including subclasses (such as +:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -170,7 +213,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, +:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -246,7 +289,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must +instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4231fb85758913..09514c89c117ad 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst From 5afe0c17ca14df430736e549542a4b85e7e7c7ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Aug 2023 22:16:23 +0200 Subject: [PATCH 235/598] gh-108223: test.pythoninfo and libregrtest log Py_NOGIL (#108238) Enable with --disable-gil --without-pydebug: $ make pythoninfo|grep NOGIL sysconfig[Py_NOGIL]: 1 $ ./python -m test ... == Python build: nogil debug ... --- Lib/test/libregrtest/utils.py | 5 +++++ Lib/test/pythoninfo.py | 1 + 2 files changed, 6 insertions(+) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fd46819fd903fe..89a149ec5d6b36 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -228,6 +228,11 @@ def get_build_info(): ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' build = [] + + # --disable-gil + if sysconfig.get_config_var('Py_NOGIL'): + build.append("nogil") + if hasattr(sys, 'gettotalrefcount'): # --with-pydebug build.append('debug') diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index e4e098dd84cfb9..ad7d5291af42f3 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -492,6 +492,7 @@ def collect_sysconfig(info_add): 'PY_STDMODULE_CFLAGS', 'Py_DEBUG', 'Py_ENABLE_SHARED', + 'Py_NOGIL', 'SHELL', 'SOABI', 'prefix', From 4b32d4f49cf1e9b49b072391256dbf5eddba4d93 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 21 Aug 2023 16:37:07 -0400 Subject: [PATCH 236/598] gh-108223: Document --disable-gil flag in configure (#108236) --- Doc/using/configure.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index f4adea82a87c38..50f60ee54fed14 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -185,6 +185,15 @@ General Options .. versionadded:: 3.11 +.. cmdoption:: --disable-gil + + Enables **experimental** support for running Python without the + :term:`global interpreter lock` (GIL). + + See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + + .. versionadded:: 3.13 + WebAssembly Options ------------------- From e6db23f66d8741db0ffc526d8fd75373a5543e3e Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 22 Aug 2023 06:22:18 +0900 Subject: [PATCH 237/598] gh-107265: Fix code_hash for ENTER_EXECUTOR case (#108188) --- Lib/test/test_capi/test_misc.py | 6 ++-- Objects/codeobject.c | 53 +++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ea0504333bab00..1cd4c56b49bba5 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2341,7 +2341,7 @@ def long_loop(): long_loop() self.assertEqual(opt.get_count(), 10) - def test_code_richcompare(self): + def test_code_restore_for_ENTER_EXECUTOR(self): def testfunc(x): i = 0 while i < x: @@ -2350,7 +2350,9 @@ def testfunc(x): opt = _testinternalcapi.get_counter_optimizer() with temporary_optimizer(opt): testfunc(1000) - self.assertEqual(testfunc.__code__, testfunc.__code__.replace()) + code, replace_code = testfunc.__code__, testfunc.__code__.replace() + self.assertEqual(code, replace_code) + self.assertEqual(hash(code), hash(replace_code)) def get_first_executor(func): diff --git a/Objects/codeobject.c b/Objects/codeobject.c index c34905c3196063..dca5804a91d2cd 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1781,30 +1781,33 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; + uint8_t co_code = co_instr.op.code; + uint8_t co_arg = co_instr.op.arg; + uint8_t cp_code = cp_instr.op.code; + uint8_t cp_arg = cp_instr.op.arg; - if (co_instr.op.code == ENTER_EXECUTOR) { - const int exec_index = co_instr.op.arg; + if (co_code == ENTER_EXECUTOR) { + const int exec_index = co_arg; _PyExecutorObject *exec = co->co_executors->executors[exec_index]; - co_instr.op.code = exec->vm_data.opcode; - co_instr.op.arg = exec->vm_data.oparg; + co_code = exec->vm_data.opcode; + co_arg = exec->vm_data.oparg; } - assert(co_instr.op.code != ENTER_EXECUTOR); - co_instr.op.code = _PyOpcode_Deopt[co_instr.op.code]; + assert(co_code != ENTER_EXECUTOR); + co_code = _PyOpcode_Deopt[co_code]; - if (cp_instr.op.code == ENTER_EXECUTOR) { - const int exec_index = cp_instr.op.arg; + if (cp_code == ENTER_EXECUTOR) { + const int exec_index = cp_arg; _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; - cp_instr.op.code = exec->vm_data.opcode; - cp_instr.op.arg = exec->vm_data.oparg; + cp_code = exec->vm_data.opcode; + cp_arg = exec->vm_data.oparg; } - assert(cp_instr.op.code != ENTER_EXECUTOR); - cp_instr.op.code = _PyOpcode_Deopt[cp_instr.op.code]; + assert(cp_code != ENTER_EXECUTOR); + cp_code = _PyOpcode_Deopt[cp_code]; - eq = co_instr.cache == cp_instr.cache; - if (!eq) { + if (co_code != cp_code || co_arg != cp_arg) { goto unequal; } - i += _PyOpcode_Caches[co_instr.op.code]; + i += _PyOpcode_Caches[co_code]; } /* compare constants */ @@ -1883,10 +1886,22 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _Py_GetBaseOpcode(co, i); - SCRAMBLE_IN(deop); - SCRAMBLE_IN(_PyCode_CODE(co)[i].op.arg); - i += _PyOpcode_Caches[deop]; + _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; + uint8_t co_code = co_instr.op.code; + uint8_t co_arg = co_instr.op.arg; + if (co_code == ENTER_EXECUTOR) { + _PyExecutorObject *exec = co->co_executors->executors[co_arg]; + assert(exec != NULL); + assert(exec->vm_data.opcode != ENTER_EXECUTOR); + co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; + co_arg = exec->vm_data.oparg; + } + else { + co_code = _Py_GetBaseOpcode(co, i); + } + SCRAMBLE_IN(co_code); + SCRAMBLE_IN(co_arg); + i += _PyOpcode_Caches[co_code]; } if ((Py_hash_t)uhash == -1) { return -2; From a1cc74c4eebc55795877eb3be019a1bec34402f8 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 21 Aug 2023 23:44:31 +0100 Subject: [PATCH 238/598] gh-107901: Fix missing line number on BACKWARD_JUMP at the end of a for loop (#108242) --- Lib/test/test_compile.py | 14 ++++++++++++++ .../2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst | 1 + Python/flowgraph.c | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 2ed8ae04961376..a470d6187aa673 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1031,6 +1031,20 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def test_lineno_of_backward_jump(self): + # Issue gh-107901 + def f(): + for i in x: + if y: + pass + + linenos = list(inst.positions.lineno + for inst in dis.get_instructions(f.__code__) + if inst.opname == 'JUMP_BACKWARD') + + self.assertTrue(len(linenos) > 0) + self.assertTrue(all(l is not None for l in linenos)) + def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles # a portion of the dictionary literal when the loop that iterates over the items diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst new file mode 100644 index 00000000000000..112e093736dd5d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst @@ -0,0 +1 @@ +Fix missing line number on :opcode:`JUMP_BACKWARD` at the end of a for loop. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 9d7865661a8036..719ed92105074e 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -531,7 +531,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label.id, NO_LOCATION); + basicblock_addop(backwards_jump, JUMP, target->b_label.id, last->i_loc); backwards_jump->b_instr[0].i_target = target; last->i_opcode = reversed_opcode; last->i_target = b->b_next; From 531930f47f6b2a548d31e62cb4ad3e215a24bf53 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 01:41:13 +0200 Subject: [PATCH 239/598] Fix test_generators: save/restore warnings filters (#108246) Previously, depending on existing filters, the test could modify the warnings and so fail as "env changed". --- Lib/test/test_generators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index a8a344ab8de48d..d48f0d47ba1962 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2258,6 +2258,7 @@ def printsolution(self, x): caught ValueError (xyz) >>> import warnings +>>> old_filters = warnings.filters.copy() >>> warnings.filterwarnings("ignore", category=DeprecationWarning) # Filter DeprecationWarning: regarding the (type, val, tb) signature of throw(). @@ -2331,8 +2332,7 @@ def printsolution(self, x): ... ValueError: 7 ->>> warnings.filters.pop(0) -('ignore', None, , None, 0) +>>> warnings.filters[:] = old_filters # Re-enable DeprecationWarning: the (type, val, tb) exception representation is deprecated, # and may be removed in a future version of Python. From de33b5c662ea8d35d81ed857c6a39e34ab94c510 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 22 Aug 2023 01:16:02 +0100 Subject: [PATCH 240/598] gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) * gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls * Update 2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst mention Windows and the former incorrect ValueError. --------- Co-authored-by: Gregory P. Smith --- Lib/ntpath.py | 12 ++++++++++++ Lib/test/test_ntpath.py | 6 ++++++ .../2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst | 4 ++++ 3 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index dadcdc0c495da1..df3402d46c9cc6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -721,6 +721,14 @@ def realpath(path, *, strict=False): try: path = _getfinalpathname(path) initial_winerror = 0 + except ValueError as ex: + # gh-106242: Raised for embedded null characters + # In strict mode, we convert into an OSError. + # Non-strict mode returns the path as-is, since we've already + # made it absolute. + if strict: + raise OSError(str(ex)) from None + path = normpath(path) except OSError as ex: if strict: raise @@ -740,6 +748,10 @@ def realpath(path, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath + except ValueError as ex: + # Unexpected, as an invalid path should not have gained a prefix + # at any point, but we ignore this error just in case. + pass except OSError as ex: # If the path does not exist and originally did not exist, then # strip the prefix anyway. diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 78e1cb582512b0..d91dcdfb0c5fac 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -394,6 +394,10 @@ def test_realpath_basic(self): d = drives.pop().encode() self.assertEqual(ntpath.realpath(d), d) + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_strict(self): @@ -404,6 +408,8 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + # gh-106242: Embedded nulls should raise OSError (not ValueError) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') diff --git a/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst new file mode 100644 index 00000000000000..ffe42ec5dc3faf --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst @@ -0,0 +1,4 @@ +Fixes :func:`~os.path.realpath` to behave consistently when passed a path +containing an embedded null character on Windows. In strict mode, it now +raises :exc:`OSError` instead of the unexpected :exc:`ValueError`, and in +non-strict mode will make the path absolute. From c965cf6dd1704a0138a4ef0a9c670e297cf66797 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 02:37:32 +0200 Subject: [PATCH 241/598] Define _Py_NULL as nullptr on C23 and newer (#108244) C23 standard added nullptr constant: https://en.wikipedia.org/wiki/C23_(C_standard_revision) --- Include/pyport.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index d7c6ae64f2bf2f..2dc241389243d3 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -24,9 +24,10 @@ #define _Py_CAST(type, expr) ((type)(expr)) // Static inline functions should use _Py_NULL rather than using directly NULL -// to prevent C++ compiler warnings. On C++11 and newer, _Py_NULL is defined as -// nullptr. -#if defined(__cplusplus) && __cplusplus >= 201103 +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) # define _Py_NULL nullptr #else # define _Py_NULL NULL From 58f9c635002673e35a56eeaa122cade7a2b6bae1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 03:16:12 +0200 Subject: [PATCH 242/598] Fix test_faulthandler for sanitizers (#108245) Set environment options to ask sanitizers to not handle SIGSEGV. This change allows running test_enable_fd() and test_enable_file() with sanitizers. Previously, they were skipped. --- Lib/test/test_faulthandler.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 2e97de592712c0..907c2cda86cbae 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -9,7 +9,6 @@ from test import support from test.support import os_helper from test.support import script_helper, is_android -from test.support import skip_if_sanitizer import tempfile import unittest from textwrap import dedent @@ -64,8 +63,20 @@ def get_output(self, code, filename=None, fd=None): pass_fds = [] if fd is not None: pass_fds.append(fd) + env = dict(os.environ) + + # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd()) + option = 'handle_segv=0' + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): + if name in env: + env[name] += f':{option}' + else: + env[name] = option + with support.SuppressCrashReport(): - process = script_helper.spawn_python('-c', code, pass_fds=pass_fds) + process = script_helper.spawn_python('-c', code, + pass_fds=pass_fds, + env=env) with process: output, stderr = process.communicate() exitcode = process.wait() @@ -304,8 +315,6 @@ def test_gil_released(self): 3, 'Segmentation fault') - @skip_if_sanitizer(memory=True, ub=True, reason="sanitizer " - "builds change crashing process output.") @skip_segfault_on_android def test_enable_file(self): with temporary_filename() as filename: @@ -321,8 +330,6 @@ def test_enable_file(self): @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") - @skip_if_sanitizer(memory=True, ub=True, reason="sanitizer " - "builds change crashing process output.") @skip_segfault_on_android def test_enable_fd(self): with tempfile.TemporaryFile('wb+') as fp: From 7f87ebbc3f52680c939791f397b9a478edf0c8d4 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 22 Aug 2023 09:37:53 +0200 Subject: [PATCH 243/598] gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) Clearly document the supported seek() operations: - Rewind to the start of the stream - Restore a previous stream position (given by tell()) - Fast-forward to the end of the stream --- Doc/library/io.rst | 16 ++++++++++++++++ Modules/_io/clinic/textio.c.h | 24 +++++++++++++++++++++--- Modules/_io/textio.c | 20 ++++++++++++++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 66273d9ed1ff0a..792bf43d9811bb 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1044,6 +1044,22 @@ Text I/O .. versionchanged:: 3.11 The method supports ``encoding="locale"`` option. + .. method:: seek(cookie, whence=os.SEEK_SET, /) + + Set the stream position. + Return the new stream position as an :class:`int`. + + Four operations are supported, + given by the following argument combinations: + + * ``seek(0, SEEK_SET)``: Rewind to the start of the stream. + * ``seek(cookie, SEEK_SET)``: Restore a previous position; + *cookie* **must be** a number returned by :meth:`!tell`. + * ``seek(0, SEEK_END)``: Fast-forward to the end of the stream. + * ``seek(0, SEEK_CUR)``: Leave the current stream position unchanged. + + Any other argument combinations are invalid, + and may raise exceptions. .. class:: StringIO(initial_value='', newline='\n') diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 5c2162529ff2dd..9bccd71e1d3ea6 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -756,9 +756,27 @@ _io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs } PyDoc_STRVAR(_io_TextIOWrapper_seek__doc__, -"seek($self, cookie, whence=0, /)\n" +"seek($self, cookie, whence=os.SEEK_SET, /)\n" "--\n" -"\n"); +"\n" +"Set the stream position, and return the new stream position.\n" +"\n" +" cookie\n" +" Zero or an opaque number returned by tell().\n" +" whence\n" +" The relative position to seek from.\n" +"\n" +"Four operations are supported, given by the following argument\n" +"combinations:\n" +"\n" +"- seek(0, SEEK_SET): Rewind to the start of the stream.\n" +"- seek(cookie, SEEK_SET): Restore a previous position;\n" +" \'cookie\' must be a number returned by tell().\n" +"- seek(0, SEEK_END): Fast-forward to the end of the stream.\n" +"- seek(0, SEEK_CUR): Leave the current stream position unchanged.\n" +"\n" +"Any other argument combinations are invalid,\n" +"and may raise exceptions."); #define _IO_TEXTIOWRAPPER_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(_io_TextIOWrapper_seek), METH_FASTCALL, _io_TextIOWrapper_seek__doc__}, @@ -957,4 +975,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=e1060638b65e8a63 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6bd981a58fcbc778 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 6ce90b2ed774c0..d87bd0054d4438 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2428,13 +2428,29 @@ _textiowrapper_encoder_setstate(textio *self, cookie_type *cookie) /*[clinic input] _io.TextIOWrapper.seek cookie as cookieObj: object - whence: int = 0 + Zero or an opaque number returned by tell(). + whence: int(c_default='0') = os.SEEK_SET + The relative position to seek from. / + +Set the stream position, and return the new stream position. + +Four operations are supported, given by the following argument +combinations: + +- seek(0, SEEK_SET): Rewind to the start of the stream. +- seek(cookie, SEEK_SET): Restore a previous position; + 'cookie' must be a number returned by tell(). +- seek(0, SEEK_END): Fast-forward to the end of the stream. +- seek(0, SEEK_CUR): Leave the current stream position unchanged. + +Any other argument combinations are invalid, +and may raise exceptions. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) -/*[clinic end generated code: output=0a15679764e2d04d input=0458abeb3d7842be]*/ +/*[clinic end generated code: output=0a15679764e2d04d input=0f68adcb02cf2823]*/ { PyObject *posobj; cookie_type cookie; From 86617518c4ac824e2b6dc20691ba5a08df04f285 Mon Sep 17 00:00:00 2001 From: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Date: Tue, 22 Aug 2023 03:41:50 -0400 Subject: [PATCH 244/598] gh-108179: Add error message for parser stack overflows (#108256) --- Lib/test/test_syntax.py | 2 +- Parser/parser.c | 1590 ++++++++-------------- Parser/pegen.h | 2 + Parser/pegen_errors.c | 8 + Tools/peg_generator/pegen/c_generator.py | 3 +- 5 files changed, 542 insertions(+), 1063 deletions(-) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f3d6cd7bad0eec..4c988382f8b411 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2335,7 +2335,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "", mode) @support.cpython_only diff --git a/Parser/parser.c b/Parser/parser.c index 44312cff125ae7..95cbef4a64a10f 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -1128,8 +1128,7 @@ static mod_ty file_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1175,8 +1174,7 @@ static mod_ty interactive_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1219,8 +1217,7 @@ static mod_ty eval_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1269,8 +1266,7 @@ static mod_ty func_type_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1331,8 +1327,7 @@ static expr_ty fstring_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1381,8 +1376,7 @@ static asdl_stmt_seq* statements_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1425,8 +1419,7 @@ static asdl_stmt_seq* statement_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1493,8 +1486,7 @@ static asdl_stmt_seq* statement_newline_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1625,8 +1617,7 @@ static asdl_stmt_seq* simple_stmts_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1719,8 +1710,7 @@ static stmt_ty simple_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2101,8 +2091,7 @@ static stmt_ty compound_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2292,8 +2281,7 @@ static stmt_ty assignment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2511,8 +2499,7 @@ static expr_ty annotated_rhs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2582,8 +2569,7 @@ static AugOperator* augassign_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2914,8 +2900,7 @@ static stmt_ty return_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2979,8 +2964,7 @@ static stmt_ty raise_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3080,8 +3064,7 @@ static stmt_ty global_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3145,8 +3128,7 @@ static stmt_ty nonlocal_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3210,8 +3192,7 @@ static stmt_ty del_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3296,8 +3277,7 @@ static stmt_ty yield_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3358,8 +3338,7 @@ static stmt_ty assert_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3426,8 +3405,7 @@ static stmt_ty import_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3503,8 +3481,7 @@ static stmt_ty import_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3570,8 +3547,7 @@ static stmt_ty import_from_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3690,8 +3666,7 @@ static asdl_alias_seq* import_from_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3826,8 +3801,7 @@ static asdl_alias_seq* import_from_as_names_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3870,8 +3844,7 @@ static alias_ty import_from_as_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3935,8 +3908,7 @@ static asdl_alias_seq* dotted_as_names_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3979,8 +3951,7 @@ static alias_ty dotted_as_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4046,8 +4017,7 @@ static expr_ty dotted_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, dotted_name_type, &_res)) { @@ -4081,8 +4051,7 @@ static expr_ty dotted_name_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4150,8 +4119,7 @@ static asdl_stmt_seq* block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4246,8 +4214,7 @@ static asdl_expr_seq* decorators_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4290,8 +4257,7 @@ static stmt_ty class_def_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4358,8 +4324,7 @@ static stmt_ty class_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4454,8 +4419,7 @@ static stmt_ty function_def_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4523,8 +4487,7 @@ static stmt_ty function_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4694,8 +4657,7 @@ static arguments_ty params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4757,8 +4719,7 @@ static arguments_ty parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4921,8 +4882,7 @@ static asdl_arg_seq* slash_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5002,8 +4962,7 @@ static SlashWithDefault* slash_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5092,8 +5051,7 @@ static StarEtc* star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5254,8 +5212,7 @@ static arg_ty kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5320,8 +5277,7 @@ static arg_ty param_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5401,8 +5357,7 @@ static arg_ty param_no_default_star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5480,8 +5435,7 @@ static NameDefaultPair* param_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5567,8 +5521,7 @@ static NameDefaultPair* param_maybe_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5652,8 +5605,7 @@ static arg_ty param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5717,8 +5669,7 @@ static arg_ty param_star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5782,8 +5733,7 @@ static expr_ty annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5829,8 +5779,7 @@ static expr_ty star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5876,8 +5825,7 @@ static expr_ty default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5945,8 +5893,7 @@ static stmt_ty if_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6086,8 +6033,7 @@ static stmt_ty elif_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6224,8 +6170,7 @@ static asdl_stmt_seq* else_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6293,8 +6238,7 @@ static stmt_ty while_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6390,8 +6334,7 @@ static stmt_ty for_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6588,8 +6531,7 @@ static stmt_ty with_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6858,8 +6800,7 @@ static withitem_ty with_item_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6957,8 +6898,7 @@ static stmt_ty try_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7147,8 +7087,7 @@ static excepthandler_ty except_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7301,8 +7240,7 @@ static excepthandler_ty except_star_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7416,8 +7354,7 @@ static asdl_stmt_seq* finally_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7487,8 +7424,7 @@ static stmt_ty match_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7586,8 +7522,7 @@ static expr_ty subject_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7673,8 +7608,7 @@ static match_case_ty case_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7748,8 +7682,7 @@ static expr_ty guard_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7795,8 +7728,7 @@ static pattern_ty patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7876,8 +7808,7 @@ static pattern_ty pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7934,8 +7865,7 @@ static pattern_ty as_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8021,8 +7951,7 @@ static pattern_ty or_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8091,8 +8020,7 @@ static pattern_ty closed_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8274,8 +8202,7 @@ static pattern_ty literal_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8509,8 +8436,7 @@ static expr_ty literal_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8698,8 +8624,7 @@ static expr_ty complex_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8805,8 +8730,7 @@ static expr_ty signed_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8889,8 +8813,7 @@ static expr_ty signed_real_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8973,8 +8896,7 @@ static expr_ty real_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9017,8 +8939,7 @@ static expr_ty imaginary_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9061,8 +8982,7 @@ static pattern_ty capture_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9123,8 +9043,7 @@ static expr_ty pattern_capture_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9171,8 +9090,7 @@ static pattern_ty wildcard_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9233,8 +9151,7 @@ static pattern_ty value_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9299,8 +9216,7 @@ static expr_ty attr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, attr_type, &_res)) { @@ -9334,8 +9250,7 @@ static expr_ty attr_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9403,8 +9318,7 @@ static expr_ty name_or_attr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9461,8 +9375,7 @@ static pattern_ty group_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9511,8 +9424,7 @@ static pattern_ty sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9618,8 +9530,7 @@ static asdl_seq* open_sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9668,8 +9579,7 @@ static asdl_seq* maybe_sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9716,8 +9626,7 @@ static pattern_ty maybe_star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9774,8 +9683,7 @@ static pattern_ty star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9884,8 +9792,7 @@ static pattern_ty mapping_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10084,8 +9991,7 @@ static asdl_seq* items_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10123,8 +10029,7 @@ static KeyPatternPair* key_value_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10173,8 +10078,7 @@ static expr_ty double_star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10225,8 +10129,7 @@ static pattern_ty class_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10456,8 +10359,7 @@ static asdl_pattern_seq* positional_patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10500,8 +10402,7 @@ static asdl_seq* keyword_patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10539,8 +10440,7 @@ static KeyPatternPair* keyword_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10589,8 +10489,7 @@ static stmt_ty type_alias_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10663,8 +10562,7 @@ static asdl_type_param_seq* type_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10713,8 +10611,7 @@ static asdl_type_param_seq* type_param_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10766,8 +10663,7 @@ static type_param_ty type_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10974,8 +10870,7 @@ static expr_ty type_param_bound_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11021,8 +10916,7 @@ static expr_ty expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11150,8 +11044,7 @@ static expr_ty expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11305,8 +11198,7 @@ static expr_ty yield_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11412,8 +11304,7 @@ static expr_ty star_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11536,8 +11427,7 @@ static expr_ty star_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11625,8 +11515,7 @@ static asdl_expr_seq* star_named_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11673,8 +11562,7 @@ static expr_ty star_named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11757,8 +11645,7 @@ static expr_ty assignment_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11832,8 +11719,7 @@ static expr_ty named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11911,8 +11797,7 @@ static expr_ty disjunction_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12000,8 +11885,7 @@ static expr_ty conjunction_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12089,8 +11973,7 @@ static expr_ty inversion_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12178,8 +12061,7 @@ static expr_ty comparison_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12272,8 +12154,7 @@ static CmpopExprPair* compare_op_bitwise_or_pair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12482,8 +12363,7 @@ static CmpopExprPair* eq_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12529,8 +12409,7 @@ static CmpopExprPair* noteq_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12576,8 +12455,7 @@ static CmpopExprPair* lte_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12623,8 +12501,7 @@ static CmpopExprPair* lt_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12670,8 +12547,7 @@ static CmpopExprPair* gte_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12717,8 +12593,7 @@ static CmpopExprPair* gt_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12764,8 +12639,7 @@ static CmpopExprPair* notin_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12814,8 +12688,7 @@ static CmpopExprPair* in_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12861,8 +12734,7 @@ static CmpopExprPair* isnot_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12911,8 +12783,7 @@ static CmpopExprPair* is_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12960,8 +12831,7 @@ static expr_ty bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_or_type, &_res)) { @@ -12995,8 +12865,7 @@ static expr_ty bitwise_or_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13084,8 +12953,7 @@ static expr_ty bitwise_xor_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_xor_type, &_res)) { @@ -13119,8 +12987,7 @@ static expr_ty bitwise_xor_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13208,8 +13075,7 @@ static expr_ty bitwise_and_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_and_type, &_res)) { @@ -13243,8 +13109,7 @@ static expr_ty bitwise_and_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13332,8 +13197,7 @@ static expr_ty shift_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, shift_expr_type, &_res)) { @@ -13367,8 +13231,7 @@ static expr_ty shift_expr_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13495,8 +13358,7 @@ static expr_ty sum_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, sum_type, &_res)) { @@ -13530,8 +13392,7 @@ static expr_ty sum_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13664,8 +13525,7 @@ static expr_ty term_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, term_type, &_res)) { @@ -13699,8 +13559,7 @@ static expr_ty term_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13942,8 +13801,7 @@ static expr_ty factor_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14103,8 +13961,7 @@ static expr_ty power_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14190,8 +14047,7 @@ static expr_ty await_primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14286,8 +14142,7 @@ static expr_ty primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, primary_type, &_res)) { @@ -14321,8 +14176,7 @@ static expr_ty primary_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14528,8 +14382,7 @@ static expr_ty slices_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14620,8 +14473,7 @@ static expr_ty slice_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14725,8 +14577,7 @@ static expr_ty atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15008,8 +14859,7 @@ static expr_ty group_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15077,8 +14927,7 @@ static expr_ty lambdef_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15148,8 +14997,7 @@ static arguments_ty lambda_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15211,8 +15059,7 @@ static arguments_ty lambda_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15377,8 +15224,7 @@ static asdl_arg_seq* lambda_slash_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15458,8 +15304,7 @@ static SlashWithDefault* lambda_slash_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15547,8 +15392,7 @@ static StarEtc* lambda_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15676,8 +15520,7 @@ static arg_ty lambda_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15742,8 +15585,7 @@ static arg_ty lambda_param_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15815,8 +15657,7 @@ static NameDefaultPair* lambda_param_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15894,8 +15735,7 @@ static NameDefaultPair* lambda_param_maybe_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15973,8 +15813,7 @@ static arg_ty lambda_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16035,8 +15874,7 @@ static expr_ty fstring_middle_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16100,8 +15938,7 @@ static expr_ty fstring_replacement_field_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16196,8 +16033,7 @@ static ResultTokenWithMetadata* fstring_conversion_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16243,8 +16079,7 @@ static ResultTokenWithMetadata* fstring_full_format_spec_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16308,8 +16143,7 @@ static expr_ty fstring_format_spec_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16371,8 +16205,7 @@ static expr_ty string_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16415,8 +16248,7 @@ static expr_ty strings_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16482,8 +16314,7 @@ static expr_ty list_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16550,8 +16381,7 @@ static expr_ty tuple_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16618,8 +16448,7 @@ static expr_ty set_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16686,8 +16515,7 @@ static expr_ty dict_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16779,8 +16607,7 @@ static asdl_seq* double_starred_kvpairs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16827,8 +16654,7 @@ static KeyValuePair* double_starred_kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16893,8 +16719,7 @@ static KeyValuePair* kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16943,8 +16768,7 @@ static asdl_comprehension_seq* for_if_clauses_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16990,8 +16814,7 @@ static comprehension_ty for_if_clause_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17118,8 +16941,7 @@ static expr_ty listcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17208,8 +17030,7 @@ static expr_ty setcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17300,8 +17121,7 @@ static expr_ty genexp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17390,8 +17210,7 @@ static expr_ty dictcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17480,8 +17299,7 @@ static expr_ty arguments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17556,8 +17374,7 @@ static expr_ty args_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17657,8 +17474,7 @@ static asdl_seq* kwargs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17745,8 +17561,7 @@ static expr_ty starred_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17829,8 +17644,7 @@ static KeywordOrStarred* kwarg_or_starred_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17940,8 +17754,7 @@ static KeywordOrStarred* kwarg_or_double_starred_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18063,8 +17876,7 @@ static expr_ty star_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18158,8 +17970,7 @@ static asdl_expr_seq* star_targets_list_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18206,8 +18017,7 @@ static asdl_expr_seq* star_targets_tuple_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18284,8 +18094,7 @@ static expr_ty star_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18376,8 +18185,7 @@ static expr_ty target_with_star_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18518,8 +18326,7 @@ static expr_ty star_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18679,8 +18486,7 @@ static expr_ty single_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18774,8 +18580,7 @@ static expr_ty single_subscript_attribute_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18895,8 +18700,7 @@ static expr_ty t_primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, t_primary_type, &_res)) { @@ -18930,8 +18734,7 @@ static expr_ty t_primary_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19152,8 +18955,7 @@ static void * t_lookahead_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19229,8 +19031,7 @@ static asdl_expr_seq* del_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19280,8 +19081,7 @@ static expr_ty del_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19418,8 +19218,7 @@ static expr_ty del_t_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19586,8 +19385,7 @@ static asdl_expr_seq* type_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19831,8 +19629,7 @@ static Token* func_type_comment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19925,8 +19722,7 @@ static void * invalid_arguments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20169,8 +19965,7 @@ static void * invalid_kwarg_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20316,8 +20111,7 @@ expression_without_invalid_rule(Parser *p) int _prev_call_invalid = p->call_invalid_rules; p->call_invalid_rules = 0; if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->call_invalid_rules = _prev_call_invalid; @@ -20436,8 +20230,7 @@ static void * invalid_legacy_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20488,8 +20281,7 @@ static void * invalid_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20605,8 +20397,7 @@ static void * invalid_named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20732,8 +20523,7 @@ static void * invalid_assignment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20938,8 +20728,7 @@ static expr_ty invalid_ann_assign_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21026,8 +20815,7 @@ static void * invalid_del_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21073,8 +20861,7 @@ static void * invalid_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21122,8 +20909,7 @@ static void * invalid_comprehension_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21241,8 +21027,7 @@ static void * invalid_dict_comprehension_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21303,8 +21088,7 @@ static void * invalid_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21521,8 +21305,7 @@ static void * invalid_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21571,8 +21354,7 @@ static void * invalid_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21714,8 +21496,7 @@ static void * invalid_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21830,8 +21611,7 @@ static void * invalid_parameters_helper_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21899,8 +21679,7 @@ static void * invalid_lambda_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22119,8 +21898,7 @@ static void * invalid_lambda_parameters_helper_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22185,8 +21963,7 @@ static void * invalid_lambda_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22301,8 +22078,7 @@ static void * invalid_lambda_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22417,8 +22193,7 @@ static void * invalid_double_type_comments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22473,8 +22248,7 @@ static void * invalid_with_item_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22525,8 +22299,7 @@ static void * invalid_for_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22576,8 +22349,7 @@ static void * invalid_group_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22659,8 +22431,7 @@ static void * invalid_import_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22712,8 +22483,7 @@ static void * invalid_import_from_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22764,8 +22534,7 @@ static void * invalid_with_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22864,8 +22633,7 @@ static void * invalid_with_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22976,8 +22744,7 @@ static void * invalid_try_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23156,8 +22923,7 @@ static void * invalid_except_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23315,8 +23081,7 @@ static void * invalid_finally_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23369,8 +23134,7 @@ static void * invalid_except_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23461,8 +23225,7 @@ static void * invalid_except_star_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23525,8 +23288,7 @@ static void * invalid_match_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23612,8 +23374,7 @@ static void * invalid_case_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23705,8 +23466,7 @@ static void * invalid_as_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23787,8 +23547,7 @@ static void * invalid_class_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23838,8 +23597,7 @@ static asdl_pattern_seq* invalid_class_argument_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23894,8 +23652,7 @@ static void * invalid_if_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23981,8 +23738,7 @@ static void * invalid_elif_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24066,8 +23822,7 @@ static void * invalid_else_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24120,8 +23875,7 @@ static void * invalid_while_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24207,8 +23961,7 @@ static void * invalid_for_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24313,8 +24066,7 @@ static void * invalid_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24388,8 +24140,7 @@ static void * invalid_class_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24484,8 +24235,7 @@ static void * invalid_double_starred_kvpairs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24594,8 +24344,7 @@ static void * invalid_kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24702,8 +24451,7 @@ static void * invalid_starred_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24766,8 +24514,7 @@ static void * invalid_replacement_field_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25127,8 +24874,7 @@ static void * invalid_conversion_character_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25199,8 +24945,7 @@ static asdl_seq * _loop0_1_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25267,8 +25012,7 @@ static asdl_seq * _loop0_2_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25335,8 +25079,7 @@ static asdl_seq * _loop0_3_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25403,8 +25146,7 @@ static asdl_seq * _loop1_4_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25476,8 +25218,7 @@ static asdl_seq * _loop0_6_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25553,8 +25294,7 @@ static asdl_seq * _gather_5_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25595,8 +25335,7 @@ static void * _tmp_7_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25653,8 +25392,7 @@ static void * _tmp_8_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25730,8 +25468,7 @@ static void * _tmp_9_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25788,8 +25525,7 @@ static void * _tmp_10_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25846,8 +25582,7 @@ static void * _tmp_11_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25904,8 +25639,7 @@ static void * _tmp_12_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25951,8 +25685,7 @@ static void * _tmp_13_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26020,8 +25753,7 @@ static void * _tmp_14_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26067,8 +25799,7 @@ static asdl_seq * _loop1_15_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26140,8 +25871,7 @@ static void * _tmp_16_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26198,8 +25928,7 @@ static void * _tmp_17_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26256,8 +25985,7 @@ static void * _tmp_18_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26303,8 +26031,7 @@ static asdl_seq * _loop0_20_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26380,8 +26107,7 @@ static asdl_seq * _gather_19_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26422,8 +26148,7 @@ static asdl_seq * _loop0_22_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26499,8 +26224,7 @@ static asdl_seq * _gather_21_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26541,8 +26265,7 @@ static void * _tmp_23_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26599,8 +26322,7 @@ static void * _tmp_24_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26646,8 +26368,7 @@ static asdl_seq * _loop0_25_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26714,8 +26435,7 @@ static asdl_seq * _loop1_26_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26787,8 +26507,7 @@ static asdl_seq * _loop0_28_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26864,8 +26583,7 @@ static asdl_seq * _gather_27_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26906,8 +26624,7 @@ static void * _tmp_29_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26953,8 +26670,7 @@ static asdl_seq * _loop0_31_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27030,8 +26746,7 @@ static asdl_seq * _gather_30_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27072,8 +26787,7 @@ static void * _tmp_32_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27119,8 +26833,7 @@ static asdl_seq * _loop1_33_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27192,8 +26905,7 @@ static void * _tmp_34_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27242,8 +26954,7 @@ static void * _tmp_35_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27289,8 +27000,7 @@ static void * _tmp_36_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27336,8 +27046,7 @@ static asdl_seq * _loop0_37_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27404,8 +27113,7 @@ static asdl_seq * _loop0_38_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27472,8 +27180,7 @@ static asdl_seq * _loop0_39_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27540,8 +27247,7 @@ static asdl_seq * _loop1_40_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27613,8 +27319,7 @@ static asdl_seq * _loop0_41_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27681,8 +27386,7 @@ static asdl_seq * _loop1_42_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27754,8 +27458,7 @@ static asdl_seq * _loop1_43_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27827,8 +27530,7 @@ static asdl_seq * _loop1_44_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27900,8 +27602,7 @@ static asdl_seq * _loop0_45_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27968,8 +27669,7 @@ static asdl_seq * _loop1_46_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28041,8 +27741,7 @@ static asdl_seq * _loop0_47_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28109,8 +27808,7 @@ static asdl_seq * _loop1_48_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28182,8 +27880,7 @@ static asdl_seq * _loop0_49_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28250,8 +27947,7 @@ static asdl_seq * _loop0_50_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28318,8 +28014,7 @@ static asdl_seq * _loop1_51_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28391,8 +28086,7 @@ static asdl_seq * _loop0_53_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28468,8 +28162,7 @@ static asdl_seq * _gather_52_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28510,8 +28203,7 @@ static asdl_seq * _loop0_55_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28587,8 +28279,7 @@ static asdl_seq * _gather_54_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28629,8 +28320,7 @@ static asdl_seq * _loop0_57_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28706,8 +28396,7 @@ static asdl_seq * _gather_56_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28748,8 +28437,7 @@ static asdl_seq * _loop0_59_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28825,8 +28513,7 @@ static asdl_seq * _gather_58_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28867,8 +28554,7 @@ static void * _tmp_60_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28944,8 +28630,7 @@ static asdl_seq * _loop1_61_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29017,8 +28702,7 @@ static asdl_seq * _loop1_62_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29090,8 +28774,7 @@ static void * _tmp_63_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29137,8 +28820,7 @@ static void * _tmp_64_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29184,8 +28866,7 @@ static asdl_seq * _loop1_65_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29257,8 +28938,7 @@ static asdl_seq * _loop0_67_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29334,8 +29014,7 @@ static asdl_seq * _gather_66_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29376,8 +29055,7 @@ static void * _tmp_68_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29434,8 +29112,7 @@ static void * _tmp_69_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29492,8 +29169,7 @@ static void * _tmp_70_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29569,8 +29245,7 @@ static void * _tmp_71_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29646,8 +29321,7 @@ static asdl_seq * _loop0_73_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29723,8 +29397,7 @@ static asdl_seq * _gather_72_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29765,8 +29438,7 @@ static asdl_seq * _loop0_75_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29842,8 +29514,7 @@ static asdl_seq * _gather_74_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29884,8 +29555,7 @@ static void * _tmp_76_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29942,8 +29612,7 @@ static asdl_seq * _loop0_78_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30019,8 +29688,7 @@ static asdl_seq * _gather_77_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30061,8 +29729,7 @@ static asdl_seq * _loop0_80_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30138,8 +29805,7 @@ static asdl_seq * _gather_79_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30180,8 +29846,7 @@ static asdl_seq * _loop0_82_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30257,8 +29922,7 @@ static asdl_seq * _gather_81_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30299,8 +29963,7 @@ static asdl_seq * _loop1_83_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30372,8 +30035,7 @@ static asdl_seq * _loop1_84_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30445,8 +30107,7 @@ static asdl_seq * _loop0_86_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30522,8 +30183,7 @@ static asdl_seq * _gather_85_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30564,8 +30224,7 @@ static asdl_seq * _loop1_87_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30637,8 +30296,7 @@ static asdl_seq * _loop1_88_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30710,8 +30368,7 @@ static asdl_seq * _loop1_89_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30783,8 +30440,7 @@ static void * _tmp_90_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30827,8 +30483,7 @@ static asdl_seq * _loop0_92_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30904,8 +30559,7 @@ static asdl_seq * _gather_91_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30946,8 +30600,7 @@ static void * _tmp_93_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30993,8 +30646,7 @@ static void * _tmp_94_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31051,8 +30703,7 @@ static void * _tmp_95_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31128,8 +30779,7 @@ static void * _tmp_96_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31186,8 +30836,7 @@ static void * _tmp_97_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31282,8 +30931,7 @@ static void * _tmp_98_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31340,8 +30988,7 @@ static asdl_seq * _loop0_99_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31408,8 +31055,7 @@ static asdl_seq * _loop0_100_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31476,8 +31122,7 @@ static asdl_seq * _loop0_101_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31544,8 +31189,7 @@ static asdl_seq * _loop1_102_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31617,8 +31261,7 @@ static asdl_seq * _loop0_103_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31685,8 +31328,7 @@ static asdl_seq * _loop1_104_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31758,8 +31400,7 @@ static asdl_seq * _loop1_105_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31831,8 +31472,7 @@ static asdl_seq * _loop1_106_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31904,8 +31544,7 @@ static asdl_seq * _loop0_107_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31972,8 +31611,7 @@ static asdl_seq * _loop1_108_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32045,8 +31683,7 @@ static asdl_seq * _loop0_109_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32113,8 +31750,7 @@ static asdl_seq * _loop1_110_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32186,8 +31822,7 @@ static asdl_seq * _loop0_111_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32254,8 +31889,7 @@ static asdl_seq * _loop1_112_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32327,8 +31961,7 @@ static void * _tmp_113_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32385,8 +32018,7 @@ static asdl_seq * _loop0_114_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32453,8 +32085,7 @@ static asdl_seq * _loop1_115_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32526,8 +32157,7 @@ static void * _tmp_116_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32576,8 +32206,7 @@ static asdl_seq * _loop0_118_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32653,8 +32282,7 @@ static asdl_seq * _gather_117_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32695,8 +32323,7 @@ static asdl_seq * _loop1_119_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32768,8 +32395,7 @@ static asdl_seq * _loop0_120_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32836,8 +32462,7 @@ static asdl_seq * _loop0_121_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32904,8 +32529,7 @@ static void * _tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32964,8 +32588,7 @@ static asdl_seq * _loop0_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33042,8 +32665,7 @@ static asdl_seq * _gather_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33084,8 +32706,7 @@ static void * _tmp_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33131,8 +32752,7 @@ static asdl_seq * _loop0_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33208,8 +32828,7 @@ static asdl_seq * _gather_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33250,8 +32869,7 @@ static asdl_seq * _loop0_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33327,8 +32945,7 @@ static asdl_seq * _gather_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33369,8 +32986,7 @@ static asdl_seq * _loop0_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33446,8 +33062,7 @@ static asdl_seq * _gather_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33488,8 +33103,7 @@ static asdl_seq * _loop0_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33565,8 +33179,7 @@ static asdl_seq * _gather_132_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33607,8 +33220,7 @@ static asdl_seq * _loop0_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33675,8 +33287,7 @@ static asdl_seq * _loop0_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33752,8 +33363,7 @@ static asdl_seq * _gather_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33794,8 +33404,7 @@ static asdl_seq * _loop1_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33867,8 +33476,7 @@ static void * _tmp_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33908,8 +33516,7 @@ static asdl_seq * _loop0_140_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33985,8 +33592,7 @@ static asdl_seq * _gather_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34027,8 +33633,7 @@ static asdl_seq * _loop0_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34104,8 +33709,7 @@ static asdl_seq * _gather_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34146,8 +33750,7 @@ static asdl_seq * _loop0_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34223,8 +33826,7 @@ static asdl_seq * _gather_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34265,8 +33867,7 @@ static asdl_seq * _loop0_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34342,8 +33943,7 @@ static asdl_seq * _gather_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34384,8 +33984,7 @@ static asdl_seq * _loop0_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34461,8 +34060,7 @@ static asdl_seq * _gather_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34503,8 +34101,7 @@ static void * _tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34545,8 +34142,7 @@ static void * _tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34606,8 +34202,7 @@ static void * _tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34648,8 +34243,7 @@ static void * _tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34706,8 +34300,7 @@ static void * _tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34783,8 +34376,7 @@ static void * _tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34825,8 +34417,7 @@ static void * _tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34886,8 +34477,7 @@ static void * _tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34944,8 +34534,7 @@ static void * _tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35002,8 +34591,7 @@ static void * _tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35060,8 +34648,7 @@ static void * _tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35194,8 +34781,7 @@ static void * _tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35252,8 +34838,7 @@ static asdl_seq * _loop0_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35320,8 +34905,7 @@ static asdl_seq * _loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35388,8 +34972,7 @@ static asdl_seq * _loop0_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35456,8 +35039,7 @@ static void * _tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35514,8 +35096,7 @@ static void * _tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35591,8 +35172,7 @@ static void * _tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35649,8 +35229,7 @@ static void * _tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35707,8 +35286,7 @@ static void * _tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35765,8 +35343,7 @@ static asdl_seq * _loop0_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35833,8 +35410,7 @@ static asdl_seq * _loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35901,8 +35477,7 @@ static asdl_seq * _loop0_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35969,8 +35544,7 @@ static asdl_seq * _loop1_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36042,8 +35616,7 @@ static void * _tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36100,8 +35673,7 @@ static asdl_seq * _loop0_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36168,8 +35740,7 @@ static void * _tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36226,8 +35797,7 @@ static asdl_seq * _loop0_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36294,8 +35864,7 @@ static asdl_seq * _loop1_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36367,8 +35936,7 @@ static void * _tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36425,8 +35993,7 @@ static void * _tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36486,8 +36053,7 @@ static void * _tmp_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36544,8 +36110,7 @@ static asdl_seq * _loop0_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36612,8 +36177,7 @@ static void * _tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36670,8 +36234,7 @@ static void * _tmp_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36747,8 +36310,7 @@ static asdl_seq * _loop1_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36820,8 +36382,7 @@ static void * _tmp_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36878,8 +36439,7 @@ static asdl_seq * _loop0_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36946,8 +36506,7 @@ static asdl_seq * _loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37014,8 +36573,7 @@ static asdl_seq * _loop0_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37082,8 +36640,7 @@ static asdl_seq * _loop0_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37159,8 +36716,7 @@ static asdl_seq * _gather_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37201,8 +36757,7 @@ static void * _tmp_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37259,8 +36814,7 @@ static asdl_seq * _loop0_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37327,8 +36881,7 @@ static void * _tmp_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37385,8 +36938,7 @@ static asdl_seq * _loop0_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37453,8 +37005,7 @@ static asdl_seq * _loop1_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37526,8 +37077,7 @@ static asdl_seq * _loop1_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37599,8 +37149,7 @@ static void * _tmp_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37660,8 +37209,7 @@ static void * _tmp_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37718,8 +37266,7 @@ static asdl_seq * _loop0_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37786,8 +37333,7 @@ static void * _tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37844,8 +37390,7 @@ static void * _tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37921,8 +37466,7 @@ static void * _tmp_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37998,8 +37542,7 @@ static asdl_seq * _loop0_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38075,8 +37618,7 @@ static asdl_seq * _gather_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38117,8 +37659,7 @@ static asdl_seq * _loop0_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38194,8 +37735,7 @@ static asdl_seq * _gather_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38236,8 +37776,7 @@ static asdl_seq * _loop0_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38313,8 +37852,7 @@ static asdl_seq * _gather_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38355,8 +37893,7 @@ static asdl_seq * _loop0_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38432,8 +37969,7 @@ static asdl_seq * _gather_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38474,8 +38010,7 @@ static asdl_seq * _loop0_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38551,8 +38086,7 @@ static asdl_seq * _gather_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38593,8 +38127,7 @@ static void * _tmp_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38651,8 +38184,7 @@ static asdl_seq * _loop0_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38719,8 +38251,7 @@ static asdl_seq * _loop1_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38792,8 +38323,7 @@ static void * _tmp_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38834,8 +38364,7 @@ static asdl_seq * _loop0_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38902,8 +38431,7 @@ static asdl_seq * _loop1_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38975,8 +38503,7 @@ static void * _tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39018,8 +38545,7 @@ static void * _tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39060,8 +38586,7 @@ static void * _tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39102,8 +38627,7 @@ static void * _tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39160,8 +38684,7 @@ static void * _tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39202,8 +38725,7 @@ static void * _tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39244,8 +38766,7 @@ static void * _tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39286,8 +38807,7 @@ static void * _tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39328,8 +38848,7 @@ static void * _tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39374,8 +38893,7 @@ static void * _tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39420,8 +38938,7 @@ static asdl_seq * _loop0_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39497,8 +39014,7 @@ static asdl_seq * _gather_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39539,8 +39055,7 @@ static void * _tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39597,8 +39112,7 @@ static void * _tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39655,8 +39169,7 @@ static void * _tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39713,8 +39226,7 @@ static void * _tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39771,8 +39283,7 @@ static void * _tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39867,8 +39378,7 @@ static void * _tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39925,8 +39435,7 @@ static void * _tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40002,8 +39511,7 @@ static void * _tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40060,8 +39568,7 @@ static void * _tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40118,8 +39625,7 @@ static void * _tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40160,8 +39666,7 @@ static void * _tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40218,8 +39723,7 @@ static void * _tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40276,8 +39780,7 @@ static void * _tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40318,8 +39821,7 @@ static asdl_seq * _loop0_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40386,8 +39888,7 @@ static void * _tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40444,8 +39945,7 @@ static void * _tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40486,8 +39986,7 @@ static void * _tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40544,8 +40043,7 @@ static void * _tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40591,8 +40089,7 @@ static void * _tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40649,8 +40146,7 @@ static void * _tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40707,8 +40203,7 @@ static void * _tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40757,8 +40252,7 @@ static void * _tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40804,8 +40298,7 @@ static void * _tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40851,8 +40344,7 @@ static void * _tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40898,8 +40390,7 @@ static void * _tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40945,8 +40436,7 @@ static void * _tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41003,8 +40493,7 @@ static void * _tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41061,8 +40550,7 @@ static void * _tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41108,8 +40596,7 @@ static void * _tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41155,8 +40642,7 @@ static void * _tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41215,8 +40701,7 @@ static void * _tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41262,8 +40747,7 @@ static void * _tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41309,8 +40793,7 @@ static void * _tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41351,8 +40834,7 @@ static void * _tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41393,8 +40875,7 @@ static void * _tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41451,8 +40932,7 @@ static void * _tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41509,8 +40989,7 @@ static void * _tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41552,8 +41031,7 @@ static void * _tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41595,8 +41073,7 @@ static void * _tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41638,8 +41115,7 @@ static void * _tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41681,8 +41157,7 @@ static void * _tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41723,8 +41198,7 @@ static void * _tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41783,8 +41257,7 @@ static void * _tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41825,8 +41298,7 @@ static void * _tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41867,8 +41339,7 @@ static void * _tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41909,8 +41380,7 @@ static void * _tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index 0852bb51d4fe72..266d5219d45a9d 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -166,6 +166,8 @@ void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va); void _Pypegen_set_syntax_error(Parser* p, Token* last_token); +void _Pypegen_stack_overflow(Parser *p); + Py_LOCAL_INLINE(void *) RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, Py_ssize_t lineno, Py_ssize_t col_offset, diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e543d40ccd8ab7..f4009367673793 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -454,3 +454,11 @@ _Pypegen_set_syntax_error(Parser* p, Token* last_token) { // generic SyntaxError we just raised if errors are found. _PyPegen_tokenize_full_source_to_check_for_errors(p); } + +void +_Pypegen_stack_overflow(Parser *p) +{ + p->error_indicator = 1; + PyErr_SetString(PyExc_MemoryError, + "Parser stack overflowed - Python source too complex to parse"); +} diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index f57b6275f671d3..301949bdae96fe 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -375,8 +375,7 @@ def __init__( def add_level(self) -> None: self.print("if (p->level++ == MAXSTACK) {") with self.indent(): - self.print("p->error_indicator = 1;") - self.print("PyErr_NoMemory();") + self.print("_Pypegen_stack_overflow(p);") self.print("}") def remove_level(self) -> None: From 1a1bfc28912a39b500c578e9f10a8a222638d411 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 22 Aug 2023 13:10:29 +0200 Subject: [PATCH 245/598] gh-105539: Emit ResourceWarning if sqlite3 database is not closed explicitly (#108015) --- Doc/library/sqlite3.rst | 6 ++++++ Doc/whatsnew/3.13.rst | 7 +++++++ Lib/test/test_sqlite3/test_dbapi.py | 6 ++++++ .../2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst | 3 +++ Modules/_sqlite/connection.c | 8 ++++++++ 5 files changed, 30 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5f1676e5ae2f56..d2745b0dbceb23 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -630,6 +630,12 @@ Connection objects * :ref:`sqlite3-connection-shortcuts` * :ref:`sqlite3-connection-context-manager` + + .. versionchanged:: 3.13 + + A :exc:`ResourceWarning` is emitted if :meth:`close` is not called before + a :class:`!Connection` object is deleted. + An SQLite database connection has the following attributes and methods: .. method:: cursor(factory=Cursor) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bfab868d1c5b62..8509e18a7d792e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -158,6 +158,13 @@ pathlib :meth:`~pathlib.Path.is_dir`. (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) +sqlite3 +------- + +* A :exc:`ResourceWarning` is now emitted if a :class:`sqlite3.Connection` + object is not :meth:`closed ` explicitly. + (Contributed by Erlend E. Aasland in :gh:`105539`.) + tkinter ------- diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index df3c2ea8d1dbda..d80ad7af3200f0 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -583,6 +583,12 @@ def test_connect_positional_arguments(self): cx.close() self.assertEqual(cm.filename, __file__) + def test_connection_resource_warning(self): + with self.assertWarns(ResourceWarning): + cx = sqlite.connect(":memory:") + del cx + gc_collect() + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst b/Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst new file mode 100644 index 00000000000000..0098c7f2438e9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst @@ -0,0 +1,3 @@ +:mod:`sqlite3` now emits an :exc:`ResourceWarning` if a +:class:`sqlite3.Connection` object is not :meth:`closed +` explicitly. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 282855f8865940..e133977b28c378 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -493,6 +493,14 @@ connection_finalize(PyObject *self) } /* Clean up if user has not called .close() explicitly. */ + if (con->db) { + if (PyErr_ResourceWarning(self, 1, "unclosed database in %R", self)) { + /* Spurious errors can appear at shutdown */ + if (PyErr_ExceptionMatches(PyExc_Warning)) { + PyErr_WriteUnraisable(self); + } + } + } if (connection_close(con) < 0) { if (teardown) { PyErr_Clear(); From c556f9a3c9af48c9af9e1f298be638553a6c886e Mon Sep 17 00:00:00 2001 From: Junya Fukuda Date: Tue, 22 Aug 2023 21:38:01 +0900 Subject: [PATCH 246/598] gh-106971: Docs: Add missing issue reference (#106992) Co-authored-by: Hugo van Kemenade --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 65ca4c332ce35f..ff651bcd047289 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1201,7 +1201,7 @@ Pending Removal in Python 3.14 is deprecated and will raise an exception in Python 3.14. * Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. + bases using the C API (:gh:`95388`). * ``__package__`` and ``__cached__`` will cease to be set or taken into consideration by the import system (:gh:`97879`). From d7202e4879bf4e7e00a69500ddcb3143864139b4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 22 Aug 2023 15:50:30 +0300 Subject: [PATCH 247/598] gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) --- Doc/c-api/exceptions.rst | 14 +++++++++----- Doc/c-api/init_config.rst | 4 ++++ Doc/c-api/module.rst | 4 ++++ Doc/c-api/unicode.rst | 2 +- Doc/extending/newtypes.rst | 2 +- Doc/tools/.nitignore | 1 - Doc/whatsnew/2.2.rst | 12 ++++++------ Doc/whatsnew/2.3.rst | 2 +- Doc/whatsnew/2.4.rst | 2 +- Doc/whatsnew/2.5.rst | 10 +++++----- Doc/whatsnew/2.6.rst | 8 ++++---- Doc/whatsnew/2.7.rst | 10 +++++----- Doc/whatsnew/3.0.rst | 4 ++-- Doc/whatsnew/3.1.rst | 4 ++-- Doc/whatsnew/3.10.rst | 4 ++-- Doc/whatsnew/3.11.rst | 26 +++++++++++++------------- Doc/whatsnew/3.2.rst | 14 +++++++------- Doc/whatsnew/3.3.rst | 4 ++-- Doc/whatsnew/3.5.rst | 4 ++-- Doc/whatsnew/3.8.rst | 16 ++++++++-------- Doc/whatsnew/3.9.rst | 12 ++++++------ Misc/NEWS.d/3.11.0a1.rst | 18 +++++++++--------- Misc/NEWS.d/3.8.0a1.rst | 4 ++-- Misc/NEWS.d/3.8.0a4.rst | 2 +- Misc/NEWS.d/3.8.0b1.rst | 2 +- Misc/NEWS.d/3.9.0a1.rst | 6 +++--- Misc/NEWS.d/3.9.0a5.rst | 4 ++-- 27 files changed, 103 insertions(+), 92 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index f1d6c995188abb..6e2ac0a40a5f1b 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -222,17 +222,21 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is decoded from the filesystem + encoding (:func:`os.fsdecode`) and passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. + Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 3ad2d435665f89..56b4ee03c1a8fd 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -82,6 +82,8 @@ PyWideStringList If *length* is non-zero, *items* must be non-``NULL`` and all strings must be non-``NULL``. + .. c:namespace:: NULL + Methods: .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) @@ -101,6 +103,8 @@ PyWideStringList Python must be preinitialized to call this function. + .. c:namespace:: PyWideStringList + Structure fields: .. c:member:: Py_ssize_t length diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 187f8419d4ee4f..979b22261efa3b 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -338,6 +338,7 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :noindex: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -372,6 +373,7 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) + :noindex: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -380,6 +382,8 @@ The available slot types are: Specifies one of the following values: + .. c:namespace:: NULL + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED The module does not support being imported in subinterpreters. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index cf965fa1676524..2a2cb1b8c458e7 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1292,7 +1292,7 @@ the user settings on the machine running the codec. Encode the Unicode object using the specified code page and return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:macro:`CP_ACP` code page to get the MBCS encoder. + :c:macro:`!CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 386b3c8f4452c3..9f166eb8a4c3ff 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -298,7 +298,7 @@ have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 09514c89c117ad..bb5515b5610951 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -20,7 +20,6 @@ Doc/c-api/structures.rst Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst -Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/glossary.rst diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 7de48a40263034..d9ead57413cbbf 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1078,17 +1078,17 @@ code, none of the changes described here will affect you very much. To upgrade an extension module to the new API, perform the following steps: -* Rename :c:func:`Py_TPFLAGS_GC` to :c:func:`PyTPFLAGS_HAVE_GC`. +* Rename :c:macro:`!Py_TPFLAGS_GC` to :c:macro:`Py_TPFLAGS_HAVE_GC`. * Use :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar` to allocate objects, and :c:func:`PyObject_GC_Del` to deallocate them. -* Rename :c:func:`PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and - :c:func:`PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. +* Rename :c:func:`!PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and + :c:func:`!PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. -* Remove :c:func:`PyGC_HEAD_SIZE` from object size calculations. +* Remove :c:macro:`!PyGC_HEAD_SIZE` from object size calculations. -* Remove calls to :c:func:`PyObject_AS_GC` and :c:func:`PyObject_FROM_GC`. +* Remove calls to :c:func:`!PyObject_AS_GC` and :c:func:`!PyObject_FROM_GC`. * A new ``et`` format sequence was added to :c:func:`PyArg_ParseTuple`; ``et`` takes both a parameter and an encoding name, and converts the parameter to the @@ -1219,7 +1219,7 @@ Some of the more notable changes are: operator, but these features were rarely used and therefore buggy. The :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` attributes are also being deprecated. At the C level, the fourth argument to - the :c:func:`PyRange_New` function, ``repeat``, has also been deprecated. + the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. * There were a bunch of patches to the dictionary implementation, mostly to fix potential core dumps if a dictionary contains objects that sneakily changed diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 2120ee49f7d6bd..ba18ce343c35d1 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1897,7 +1897,7 @@ Changes to Python's build process and to the C API include: but will also mean that you can't get help for Python's built-ins. (Contributed by Gustavo Niemeyer.) -* The :c:func:`PyArg_NoArgs` macro is now deprecated, and code that uses it +* The :c:func:`!PyArg_NoArgs` macro is now deprecated, and code that uses it should be changed. For Python 2.2 and later, the method definition table can specify the :c:macro:`METH_NOARGS` flag, signalling that there are no arguments, and the argument checking can then be removed. If compatibility with pre-2.2 diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 40b404f87e5152..cab321c3e54d18 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1468,7 +1468,7 @@ Some of the changes to Python's build process and to the C API are: *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new - :c:func:`PyEval_ThreadsInitialized` function to tell if any thread operations + :c:func:`!PyEval_ThreadsInitialized` function to tell if any thread operations have been performed. If this function returns false, no lock operations are needed. (Contributed by Nick Coghlan.) diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index a47327d15fd79a..2df6d603207a10 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -2119,9 +2119,9 @@ Changes to Python's build process and to the C API include: the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and - :c:func:`PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take + :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. - This AST can then be turned into a code object by :c:func:`PyAST_Compile`. For + This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For more information, read the source code, and then ask questions on python-dev. The AST code was developed under Jeremy Hylton's management, and implemented by @@ -2172,7 +2172,7 @@ Changes to Python's build process and to the C API include: ``Py_LOCAL(type)`` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the - function be inlined. If :c:func:`PY_LOCAL_AGGRESSIVE` is defined before + function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at @@ -2181,7 +2181,7 @@ Changes to Python's build process and to the C API include: * ``PyErr_NewException(name, base, dict)`` can now accept a tuple of base classes as its *base* argument. (Contributed by Georg Brandl.) -* The :c:func:`PyErr_Warn` function for issuing warnings is now deprecated in +* The :c:func:`!PyErr_Warn` function for issuing warnings is now deprecated in favour of ``PyErr_WarnEx(category, message, stacklevel)`` which lets you specify the number of stack frames separating this function and the caller. A *stacklevel* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the @@ -2191,7 +2191,7 @@ Changes to Python's build process and to the C API include: compiled with a C++ compiler without errors. (Implemented by Anthony Baxter, Martin von Löwis, Skip Montanaro.) -* The :c:func:`PyRange_New` function was removed. It was never documented, never +* The :c:func:`!PyRange_New` function was removed. It was never documented, never used in the core code, and had dangerously lax error checking. In the unlikely case that your extensions were using it, you can replace it by something like the following:: diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 5c46dc8952154b..8b3d3a324f68f6 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -977,7 +977,7 @@ can be used to include Unicode characters:: print len(s) # 12 Unicode characters At the C level, Python 3.0 will rename the existing 8-bit -string type, called :c:type:`PyStringObject` in Python 2.x, +string type, called :c:type:`!PyStringObject` in Python 2.x, to :c:type:`PyBytesObject`. Python 2.6 uses ``#define`` to support using the names :c:func:`PyBytesObject`, :c:func:`PyBytes_Check`, :c:func:`PyBytes_FromStringAndSize`, @@ -3012,11 +3012,11 @@ Changes to Python's build process and to the C API include: bug occurred if one thread closed a file object while another thread was reading from or writing to the object. In 2.6 file objects have a reference count, manipulated by the - :c:func:`PyFile_IncUseCount` and :c:func:`PyFile_DecUseCount` + :c:func:`!PyFile_IncUseCount` and :c:func:`!PyFile_DecUseCount` functions. File objects can't be closed unless the reference count - is zero. :c:func:`PyFile_IncUseCount` should be called while the GIL + is zero. :c:func:`!PyFile_IncUseCount` should be called while the GIL is still held, before carrying out an I/O operation using the - ``FILE *`` pointer, and :c:func:`PyFile_DecUseCount` should be called + ``FILE *`` pointer, and :c:func:`!PyFile_DecUseCount` should be called immediately after the GIL is re-acquired. (Contributed by Antoine Pitrou and Gregory P. Smith.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 00001f92b51c82..37e2d04d516ab0 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2152,7 +2152,7 @@ Changes to Python's build process and to the C API include: * New function: stemming from the rewrite of string-to-float conversion, a new :c:func:`PyOS_string_to_double` function was added. The old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. * New function: :c:func:`PySys_SetArgvEx` sets the value of @@ -2195,13 +2195,13 @@ Changes to Python's build process and to the C API include: .. XXX these macros don't seem to be described in the c-api docs. -* Removed function: :c:macro:`PyEval_CallObject` is now only available +* Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be deleted by now. (Removed by Antoine Pitrou; :issue:`8276`.) -* New format codes: the :c:func:`PyFormat_FromString`, - :c:func:`PyFormat_FromStringV`, and :c:func:`PyErr_Format` functions now +* New format codes: the :c:func:`!PyString_FromFormat`, + :c:func:`!PyString_FromFormatV`, and :c:func:`PyErr_Format` functions now accept ``%lld`` and ``%llu`` format codes for displaying C's :c:expr:`long long` types. (Contributed by Mark Dickinson; :issue:`7228`.) @@ -2540,7 +2540,7 @@ For C extensions: instead of triggering a :exc:`DeprecationWarning` (:issue:`5080`). * Use the new :c:func:`PyOS_string_to_double` function instead of the old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions, + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions, which are now deprecated. For applications that embed Python: diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 58d42bd94cb61e..b0c2529e780213 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -865,8 +865,8 @@ to the C API. * No more C API support for restricted execution. -* :c:func:`PyNumber_Coerce`, :c:func:`PyNumber_CoerceEx`, - :c:func:`PyMember_Get`, and :c:func:`PyMember_Set` C APIs are removed. +* :c:func:`!PyNumber_Coerce`, :c:func:`!PyNumber_CoerceEx`, + :c:func:`!PyMember_Get`, and :c:func:`!PyMember_Set` C APIs are removed. * New C API :c:func:`PyImport_ImportModuleNoBlock`, works like :c:func:`PyImport_ImportModule` but won't block on the import lock diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 3c1c9c3c4bc601..e237179f4b1829 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -501,12 +501,12 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson and Lisandro Dalcrin; :issue:`5175`.) -* Deprecated :c:func:`PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. +* Deprecated :c:func:`!PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. (Contributed by Mark Dickinson; :issue:`4910`.) * Added a new :c:func:`PyOS_string_to_double` function to replace the - deprecated functions :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof`. + deprecated functions :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof`. (Contributed by Mark Dickinson; :issue:`5914`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index e4ca3c4c81ba58..1e6e0befa9819a 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1819,8 +1819,8 @@ Removed into their code. (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. +* Removed the :c:func:`!PyModule_GetWarningsModule` function that was useless + now due to the :mod:`!_warnings` module was converted to a builtin module in 2.6. (Contributed by Hai Shi in :issue:`42599`.) * Remove deprecated aliases to :ref:`collections-abstract-base-classes` from diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index fce00d37555c2e..58ce881e9787f6 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2216,7 +2216,7 @@ New Features * :c:func:`PyBuffer_SizeFromFormat` * :c:func:`PyBuffer_ToContiguous` * :c:func:`PyBuffer_FromContiguous` - * :c:func:`PyBuffer_CopyData` + * :c:func:`PyObject_CopyData` * :c:func:`PyBuffer_IsContiguous` * :c:func:`PyBuffer_FillContiguousStrides` * :c:func:`PyBuffer_FillInfo` @@ -2562,18 +2562,18 @@ Deprecated * Deprecate the following functions to configure the Python initialization: - * :c:func:`PySys_AddWarnOptionUnicode` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` - * :c:func:`PySys_HasWarnOptions` - * :c:func:`PySys_SetArgvEx` - * :c:func:`PySys_SetArgv` - * :c:func:`PySys_SetPath` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`_Py_SetProgramFullPath` + * :c:func:`!PySys_AddWarnOptionUnicode` + * :c:func:`!PySys_AddWarnOption` + * :c:func:`!PySys_AddXOption` + * :c:func:`!PySys_HasWarnOptions` + * :c:func:`!PySys_SetArgvEx` + * :c:func:`!PySys_SetArgv` + * :c:func:`!PySys_SetPath` + * :c:func:`!Py_SetPath` + * :c:func:`!Py_SetProgramName` + * :c:func:`!Py_SetPythonHome` + * :c:func:`!Py_SetStandardStreamEncoding` + * :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 56ac5c4c0a1c1b..df32b76b6d7b03 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2569,7 +2569,7 @@ Changes to Python's build process and to the C API include: to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). -* :c:macro:`PyEval_CallObject` is now only available in macro form. The +* :c:func:`!PyEval_CallObject` is now only available in macro form. The function declaration, which was kept for backwards compatibility reasons, is now removed -- the macro was introduced in 1997 (:issue:`8276`). @@ -2731,15 +2731,15 @@ require changes to your code: (Contributed by Antoine Pitrou, :issue:`10272`.) -* The misleading functions :c:func:`PyEval_AcquireLock()` and - :c:func:`PyEval_ReleaseLock()` have been officially deprecated. The - thread-state aware APIs (such as :c:func:`PyEval_SaveThread()` - and :c:func:`PyEval_RestoreThread()`) should be used instead. +* The misleading functions :c:func:`!PyEval_AcquireLock` and + :c:func:`!PyEval_ReleaseLock` have been officially deprecated. The + thread-state aware APIs (such as :c:func:`PyEval_SaveThread` + and :c:func:`PyEval_RestoreThread`) should be used instead. * Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and a new function, :func:`asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) -* Due to the new :term:`GIL` implementation, :c:func:`PyEval_InitThreads()` - cannot be called before :c:func:`Py_Initialize()` anymore. +* Due to the new :term:`GIL` implementation, :c:func:`!PyEval_InitThreads` + cannot be called before :c:func:`Py_Initialize` anymore. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 7e12c279065675..3f98c82c2fa556 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -2304,7 +2304,7 @@ Functions and macros manipulating Py_UNICODE* strings: Encoders: -* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_Encode`: use :c:func:`!PyUnicode_AsEncodedObject` * :c:func:`!PyUnicode_EncodeUTF7` * :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` @@ -2462,7 +2462,7 @@ Porting C code -------------- * In the course of changes to the buffer API the undocumented - :c:member:`~Py_buffer.smalltable` member of the + :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the layout of the :c:type:`PyMemoryViewObject` has changed. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 86bfdc4d478a02..0c45a42d1a7c17 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2512,7 +2512,7 @@ Changes in the Python API Changes in the C API -------------------- -* The undocumented :c:member:`~PyMemoryViewObject.format` member of the +* The undocumented :c:member:`!format` member of the (non-public) :c:type:`PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. @@ -2520,7 +2520,7 @@ Changes in the C API * The :c:type:`PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. -* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references. +* Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index c5fb5c53dfe350..7946bf910af2c4 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1574,12 +1574,12 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` - * Private functions: :c:func:`_PyObject_GC_TRACK`, - :c:func:`_PyObject_GC_UNTRACK`, :c:func:`_Py_Dealloc` + * Private functions: :c:func:`!_PyObject_GC_TRACK`, + :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` (Contributed by Victor Stinner in :issue:`35059`.) -* The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +* The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. (Contributed by Victor Stinner in :issue:`35713`.) @@ -1628,7 +1628,7 @@ Build and C API Changes parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) -* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +* :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). (Contributed by Victor Stinner in :issue:`38234`.) @@ -1845,11 +1845,11 @@ Changes in Python behavior always use ``sys.platform.startswith('aix')``. (Contributed by M. Felt in :issue:`36588`.) -* :c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +* :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. If this - behavior is not desired, guard the call by checking :c:func:`_Py_IsFinalizing` + behavior is not desired, guard the call by checking :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing`. (Contributed by Joannah Nanjekye in :issue:`36475`.) @@ -2021,7 +2021,7 @@ Changes in the C API *cf_flags*. (Contributed by Guido van Rossum in :issue:`35766`.) -* The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +* The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. (Contributed by Victor Stinner in :issue:`36728`.) @@ -2121,7 +2121,7 @@ Changes in the C API (Contributed by Antoine Pitrou in :issue:`32388`.) -* The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept +* The functions :c:func:`!PyNode_AddChild` and :c:func:`!PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. * The :file:`libpython38.a` file to allow MinGW tools to link directly against diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 5891d9a55534fd..8e2df19419bfc2 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -870,9 +870,9 @@ Deprecated users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. -* The Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` +* The Public C API functions :c:func:`!PyParser_SimpleParseStringFlags`, + :c:func:`!PyParser_SimpleParseStringFlagsFilename`, + :c:func:`!PyParser_SimpleParseFileFlags` and :c:func:`!PyNode_Compile` are deprecated and will be removed in Python 3.10 together with the old parser. * Using :data:`NotImplemented` in a boolean context has been deprecated, @@ -923,10 +923,10 @@ Deprecated (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969` and Serhiy Storchaka in :issue:`39988`.) -* The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized` +* The :c:func:`!PyEval_InitThreads` and :c:func:`!PyEval_ThreadsInitialized` functions are now deprecated and will be removed in Python 3.11. Calling - :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized - by :c:func:`Py_Initialize()` since Python 3.7. + :c:func:`!PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized + by :c:func:`Py_Initialize` since Python 3.7. (Contributed by Victor Stinner in :issue:`39877`.) * Passing ``None`` as the first argument to the :func:`shlex.split` function diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 17ee5138dbf90f..e1d0adc478029f 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -5036,15 +5036,15 @@ Limited API. Deprecate the following functions to configure the Python initialization: -* :c:func:`PySys_AddWarnOptionUnicode` -* :c:func:`PySys_AddWarnOption` -* :c:func:`PySys_AddXOption` -* :c:func:`PySys_HasWarnOptions` -* :c:func:`Py_SetPath` -* :c:func:`Py_SetProgramName` -* :c:func:`Py_SetPythonHome` -* :c:func:`Py_SetStandardStreamEncoding` -* :c:func:`_Py_SetProgramFullPath` +* :c:func:`!PySys_AddWarnOptionUnicode` +* :c:func:`!PySys_AddWarnOption` +* :c:func:`!PySys_AddXOption` +* :c:func:`!PySys_HasWarnOptions` +* :c:func:`!Py_SetPath` +* :c:func:`!Py_SetProgramName` +* :c:func:`!Py_SetPythonHome` +* :c:func:`!Py_SetStandardStreamEncoding` +* :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 467e992780382f..dbbfb6e8b0d68e 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -8765,7 +8765,7 @@ for relative path to files in current directory. .. nonce: fmehdG .. section: C API -The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. @@ -8836,7 +8836,7 @@ Py_LIMITED_API. Patch by Arthur Neufeld. .. nonce: gFd85N .. section: C API -The :c:func:`_PyObject_GC_TRACK` and :c:func:`_PyObject_GC_UNTRACK` macros +The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros have been removed from the public C API. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 524a05a7ae9704..2ce60f39539e8c 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -124,7 +124,7 @@ Galindo. .. nonce: CjRps3 .. section: Core and Builtins -:c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +:c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 72e6b31874e373..c5e27b04ef8e8b 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -2047,6 +2047,6 @@ unbound methods. These are objects supporting the optimization given by the .. nonce: FR-dMP .. section: C API -The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 43676836478c91..5a4431b0fcf1c6 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5535,7 +5535,7 @@ Tyler Kieft. .. nonce: d0bhEA .. section: C API -:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +:c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). @@ -5546,8 +5546,8 @@ path (:c:func:`Py_GetProgramFullPath`) rather than to the program name .. nonce: ZbquVK .. section: C API -Python ignored arguments passed to :c:func:`Py_SetPath`, -:c:func:`Py_SetPythonHome` and :c:func:`Py_SetProgramName`: fix Python +Python ignored arguments passed to :c:func:`!Py_SetPath`, +:c:func:`!Py_SetPythonHome` and :c:func:`!Py_SetProgramName`: fix Python initialization to use specified arguments. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index 8a1219501e81bf..19ad20ad3db042 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -1234,8 +1234,8 @@ method name in the SystemError "bad call flags" error message to ease debug. .. nonce: GOYtIm .. section: C API -Deprecated :c:func:`PyEval_InitThreads` and -:c:func:`PyEval_ThreadsInitialized`. Calling :c:func:`PyEval_InitThreads` +Deprecated :c:func:`!PyEval_InitThreads` and +:c:func:`!PyEval_ThreadsInitialized`. Calling :c:func:`!PyEval_InitThreads` now does nothing. .. From 893215a4e7f59eabb8ccdf188c4b9b1de5bd8966 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 22 Aug 2023 15:20:58 +0200 Subject: [PATCH 248/598] Docs: align the param spec of sqlite3.Connection methods with the implementation (#108285) - no parameters of create_aggregate() are positional-only - all parameters of create_collation() are positional-only --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d2745b0dbceb23..0b53f97b284f07 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -764,7 +764,7 @@ Connection objects ('acbd18db4cc2f85cedef654fccc4a4d8',) - .. method:: create_aggregate(name, /, n_arg, aggregate_class) + .. method:: create_aggregate(name, n_arg, aggregate_class) Create or remove a user-defined SQL aggregate function. @@ -904,7 +904,7 @@ Connection objects [('a', 9), ('b', 12), ('c', 16), ('d', 12), ('e', 9)] - .. method:: create_collation(name, callable) + .. method:: create_collation(name, callable, /) Create a collation named *name* using the collating function *callable*. *callable* is passed two :class:`string ` arguments, From a541e01537cc07b326cc37cc29412b816bc2b4fc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 15:52:32 +0200 Subject: [PATCH 249/598] gh-90791: Enable test___all__ on ASAN build (#108286) * Only skip modules and tests related to X11 on ASAN builds: run other tests with ASAN. * Use print(flush=True) to see output earlier when it's redirected to a pipe. * Update issue reference: replace bpo-46633 with gh-90791. --- Lib/test/test___all__.py | 33 +++++++++++++++++++------ Lib/test/test_concurrent_futures.py | 2 +- Lib/test/test_idle.py | 1 + Lib/test/test_peg_generator/__init__.py | 2 +- Lib/test/test_tkinter/__init__.py | 1 + Lib/test/test_tools/__init__.py | 2 +- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 2163f1636566b7..c87cde4b3d1fab 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -12,10 +12,19 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: test___all__ is skipped because importing some modules - # directly can trigger known problems with ASAN (like tk). - raise unittest.SkipTest("workaround ASAN build issues on loading tests " - "like tk") + SKIP_MODULES = frozenset(( + # gh-90791: Tests involving libX11 can SEGFAULT on ASAN/MSAN builds. + # Skip modules, packages and tests using '_tkinter'. + '_tkinter', + 'tkinter', + 'test_tkinter', + 'test_ttk', + 'test_ttk_textonly', + 'idlelib', + 'test_idle', + )) +else: + SKIP_MODULES = () class NoAll(RuntimeError): @@ -83,15 +92,23 @@ def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): path = os.path.join(basedir, fn) if os.path.isdir(path): + if fn in SKIP_MODULES: + continue pkg_init = os.path.join(path, '__init__.py') if os.path.exists(pkg_init): yield pkg_init, modpath + fn for p, m in self.walk_modules(path, modpath + fn + "."): yield p, m continue - if not fn.endswith('.py') or fn == '__init__.py': + + if fn == '__init__.py': + continue + if not fn.endswith('.py'): + continue + modname = fn.removesuffix('.py') + if modname in SKIP_MODULES: continue - yield path, modpath + fn[:-3] + yield path, modpath + modname def test_all(self): # List of denied modules and packages @@ -119,14 +136,14 @@ def test_all(self): if denied: continue if support.verbose: - print(modname) + print(f"Check {modname}", flush=True) try: # This heuristic speeds up the process by removing, de facto, # most test modules (and avoiding the auto-executing ones). with open(path, "rb") as f: if b"__all__" not in f.read(): raise NoAll(modname) - self.check_all(modname) + self.check_all(modname) except NoAll: ignored.append(modname) except FailedImport: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 39dbe234e765e8..0f0ea6190def06 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -34,7 +34,7 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: Skip the test because it is too slow when Python is built + # gh-90791: Skip the test because it is too slow when Python is built # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. raise unittest.SkipTest("test too slow on ASAN/MSAN build") diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index ebb1e8eb823b51..3d8b7ecc0ecb6d 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -3,6 +3,7 @@ from test.support import check_sanitizer if check_sanitizer(address=True, memory=True): + # See gh-90791 for details raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test_idle if _tkinter, tkinter, or idlelib are missing. diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index 77f72fcc7c6e3b..87281eb5e03c7f 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -5,7 +5,7 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: Skip the test because it is too slow when Python is built + # gh-90791: Skip the test because it is too slow when Python is built # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. raise unittest.SkipTest("test too slow on ASAN/MSAN build") diff --git a/Lib/test/test_tkinter/__init__.py b/Lib/test/test_tkinter/__init__.py index b1181bc04b7953..aa196c12c804ac 100644 --- a/Lib/test/test_tkinter/__init__.py +++ b/Lib/test/test_tkinter/__init__.py @@ -10,6 +10,7 @@ if check_sanitizer(address=True, memory=True): + # See gh-90791 for details raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test if _tkinter wasn't built. diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index 2102b9fceff568..dde5d84e9edc6b 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -8,7 +8,7 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: Skip the test because it is too slow when Python is built + # gh-90791: Skip the test because it is too slow when Python is built # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. raise unittest.SkipTest("test too slow on ASAN/MSAN build") From 6541fe4ad7b96ab96ee5c596b60814a93346dd27 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 16:28:20 +0200 Subject: [PATCH 250/598] Ignore _Py_write_noraise() result: cast to (void) (#108291) Code using _Py_write_noraise() usually cannot report. Ignore errors is the least surprising behavior for users. --- Modules/faulthandler.c | 4 ++-- Python/pylifecycle.c | 2 +- Python/traceback.c | 10 +++++----- Python/tracemalloc.c | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 428b090193f093..d8cfc13a7dc6d5 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -26,7 +26,7 @@ /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) -#define PUTS(fd, str) _Py_write_noraise(fd, str, strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, strlen(str)) // clang uses __attribute__((no_sanitize("undefined"))) @@ -576,7 +576,7 @@ faulthandler_thread(void *unused) /* Timeout => dump traceback */ assert(st == PY_LOCK_FAILURE); - _Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); + (void)_Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL); ok = (errmsg == NULL); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 263ead39fa472a..9837e0f0d52e6d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -57,7 +57,7 @@ # undef BYTE #endif -#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) #ifdef __cplusplus diff --git a/Python/traceback.c b/Python/traceback.c index bddb8763a2f9be..61ace38275355c 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -25,7 +25,7 @@ #define OFF(x) offsetof(PyTracebackObject, x) -#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) #define MAX_STRING_LENGTH 500 #define MAX_FRAME_DEPTH 100 #define MAX_NTHREADS 100 @@ -1047,7 +1047,7 @@ _Py_DumpDecimal(int fd, size_t value) value /= 10; } while (value); - _Py_write_noraise(fd, ptr, end - ptr); + (void)_Py_write_noraise(fd, ptr, end - ptr); } /* Format an integer as hexadecimal with width digits into fd file descriptor. @@ -1072,7 +1072,7 @@ _Py_DumpHexadecimal(int fd, uintptr_t value, Py_ssize_t width) value >>= 4; } while ((end - ptr) < width || value); - _Py_write_noraise(fd, ptr, end - ptr); + (void)_Py_write_noraise(fd, ptr, end - ptr); } void @@ -1125,7 +1125,7 @@ _Py_DumpASCII(int fd, PyObject *text) } if (!need_escape) { // The string can be written with a single write() syscall - _Py_write_noraise(fd, str, size); + (void)_Py_write_noraise(fd, str, size); goto done; } } @@ -1135,7 +1135,7 @@ _Py_DumpASCII(int fd, PyObject *text) if (' ' <= ch && ch <= 126) { /* printable ASCII character */ char c = (char)ch; - _Py_write_noraise(fd, &c, 1); + (void)_Py_write_noraise(fd, &c, 1); } else if (ch <= 0xff) { PUTS(fd, "\\x"); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index f8ad939dccaccd..7d294ea5fe744c 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -1247,7 +1247,7 @@ tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) } -#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) static void _PyMem_DumpFrame(int fd, frame_t * frame) From e8ef0bdd8c613a722bf7965bf1da912882141a52 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 22 Aug 2023 17:34:18 +0300 Subject: [PATCH 251/598] gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) --- Doc/library/enum.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index e9c4f0e2c5f59b..7653865f0b9b36 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -241,6 +241,10 @@ Data Types >>> list(reversed(Color)) [, , ] + .. versionadded:: 3.11 + + Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. + .. class:: Enum From adfc118fdab66882599e01a84c22bd897055f3f1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 16:53:49 +0200 Subject: [PATCH 252/598] gh-106016: Add Lib/test/test_module/ directory (#108293) * Move Python scripts related to test_module to this new directory: good_getattr.py and bad_getattrX.py scripts. * Move Lib/test/test_module.py to Lib/test/test_module/__init__.py. --- .../__init__.py} | 36 +++++++++---------- Lib/test/{ => test_module}/bad_getattr.py | 0 Lib/test/{ => test_module}/bad_getattr2.py | 0 Lib/test/{ => test_module}/bad_getattr3.py | 0 Lib/test/{ => test_module}/good_getattr.py | 0 Makefile.pre.in | 1 + 6 files changed, 19 insertions(+), 18 deletions(-) rename Lib/test/{test_module.py => test_module/__init__.py} (92%) rename Lib/test/{ => test_module}/bad_getattr.py (100%) rename Lib/test/{ => test_module}/bad_getattr2.py (100%) rename Lib/test/{ => test_module}/bad_getattr3.py (100%) rename Lib/test/{ => test_module}/good_getattr.py (100%) diff --git a/Lib/test/test_module.py b/Lib/test/test_module/__init__.py similarity index 92% rename from Lib/test/test_module.py rename to Lib/test/test_module/__init__.py index c7eb92290e1b6d..cfc4d9ccf1cc81 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module/__init__.py @@ -126,8 +126,8 @@ def test_weakref(self): self.assertIs(wr(), None) def test_module_getattr(self): - import test.good_getattr as gga - from test.good_getattr import test + import test.test_module.good_getattr as gga + from test.test_module.good_getattr import test self.assertEqual(test, "There is test") self.assertEqual(gga.x, 1) self.assertEqual(gga.y, 2) @@ -135,46 +135,46 @@ def test_module_getattr(self): "Deprecated, use whatever instead"): gga.yolo self.assertEqual(gga.whatever, "There is whatever") - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_getattr_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 self.assertEqual(bga.x, 1) self.assertEqual(bad_getattr2.x, 1) with self.assertRaises(TypeError): bga.nope with self.assertRaises(TypeError): bad_getattr2.nope - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_dir(self): - import test.good_getattr as gga + import test.test_module.good_getattr as gga self.assertEqual(dir(gga), ['a', 'b', 'c']) - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_dir_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 with self.assertRaises(TypeError): dir(bga) with self.assertRaises(TypeError): dir(bad_getattr2) - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_getattr_tricky(self): - from test import bad_getattr3 + from test.test_module import bad_getattr3 # these lookups should not crash with self.assertRaises(AttributeError): bad_getattr3.one with self.assertRaises(AttributeError): bad_getattr3.delgetattr - if 'test.bad_getattr3' in sys.modules: - del sys.modules['test.bad_getattr3'] + if 'test.test_module.bad_getattr3' in sys.modules: + del sys.modules['test.test_module.bad_getattr3'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ diff --git a/Lib/test/bad_getattr.py b/Lib/test/test_module/bad_getattr.py similarity index 100% rename from Lib/test/bad_getattr.py rename to Lib/test/test_module/bad_getattr.py diff --git a/Lib/test/bad_getattr2.py b/Lib/test/test_module/bad_getattr2.py similarity index 100% rename from Lib/test/bad_getattr2.py rename to Lib/test/test_module/bad_getattr2.py diff --git a/Lib/test/bad_getattr3.py b/Lib/test/test_module/bad_getattr3.py similarity index 100% rename from Lib/test/bad_getattr3.py rename to Lib/test/test_module/bad_getattr3.py diff --git a/Lib/test/good_getattr.py b/Lib/test/test_module/good_getattr.py similarity index 100% rename from Lib/test/good_getattr.py rename to Lib/test/test_module/good_getattr.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 9be5c3b50eb9ee..beccab45a235a5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2212,6 +2212,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ test/test_json \ + test/test_module \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ From b8f96b5eda5b376b05a9dbf046208388249e30a6 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 22 Aug 2023 08:29:49 -0700 Subject: [PATCH 253/598] gh-108253: Fix bug in func version cache (#108296) When a function object changed its version, a stale pointer might remain in the cache. Zap these whenever `func_version` changes (even when set to 0). --- Objects/funcobject.c | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 33191d23f18230..2820e47bdf2e6b 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -131,7 +131,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + _PyFunction_SetVersion(op, 0); _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return op; @@ -207,7 +207,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + _PyFunction_SetVersion(op, 0); _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; @@ -268,9 +268,17 @@ code objects have been created during the process's lifetime. void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (func->func_version != 0) { + PyFunctionObject **slot = + interp->func_state.func_version_cache + + (func->func_version % FUNC_VERSION_CACHE_SIZE); + if (*slot == func) { + *slot = NULL; + } + } func->func_version = version; if (version != 0) { - PyInterpreterState *interp = _PyInterpreterState_GET(); interp->func_state.func_version_cache[ version % FUNC_VERSION_CACHE_SIZE] = func; } @@ -370,7 +378,7 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, (PyFunctionObject *) op, defaults); - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_defaults, defaults); return 0; } @@ -379,7 +387,7 @@ void PyFunction_SetVectorcall(PyFunctionObject *func, vectorcallfunc vectorcall) { assert(func != NULL); - func->func_version = 0; + _PyFunction_SetVersion(func, 0); func->vectorcall = vectorcall; } @@ -412,7 +420,7 @@ PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, (PyFunctionObject *) op, defaults); - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_kwdefaults, defaults); return 0; } @@ -445,7 +453,7 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) Py_TYPE(closure)->tp_name); return -1; } - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_closure, closure); return 0; } @@ -507,7 +515,7 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); return 0; } @@ -566,7 +574,7 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) return -1; } handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } @@ -646,7 +654,7 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, op, value); - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_defaults, Py_XNewRef(value)); return 0; } @@ -687,7 +695,7 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, op, value); - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_kwdefaults, Py_XNewRef(value)); return 0; } @@ -717,7 +725,7 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno "__annotations__ must be set to a dict object"); return -1; } - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_annotations, Py_XNewRef(value)); return 0; } @@ -881,7 +889,7 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, static int func_clear(PyFunctionObject *op) { - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_CLEAR(op->func_globals); Py_CLEAR(op->func_builtins); Py_CLEAR(op->func_module); @@ -917,15 +925,7 @@ func_dealloc(PyFunctionObject *op) if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } - if (op->func_version != 0) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyFunctionObject **slot = - interp->func_state.func_version_cache - + (op->func_version % FUNC_VERSION_CACHE_SIZE); - if (*slot == op) { - *slot = NULL; - } - } + _PyFunction_SetVersion(op, 0); (void)func_clear(op); // These aren't cleared by func_clear(). Py_DECREF(op->func_code); From d2879f2095abd5c8186c7f69c964a341c2053572 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 19:12:51 +0200 Subject: [PATCH 254/598] gh-108303: Remove unused Lib/test/sgml_input.html (#108305) In Python 2.7, the file was used by Lib/test/test_sgmllib.py to test Lib/sgmllib.py. The sgmllib module and its tests have been removed in Python 3.0. --- Lib/test/sgml_input.html | 212 --------------------------------------- 1 file changed, 212 deletions(-) delete mode 100644 Lib/test/sgml_input.html diff --git a/Lib/test/sgml_input.html b/Lib/test/sgml_input.html deleted file mode 100644 index f4d2e6cc805e6a..00000000000000 --- a/Lib/test/sgml_input.html +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - - - -
- - - - - -
-
- - - - - -
- - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - -
  MetalCristalDeuterioEnerga  
160.6363.40639.230-80/3.965
-
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Flotas (max. 9)
Num.MisinCantidadComienzoSalidaObjetivoLlegadaOrden
1 - Espionaje - (F) - 3[2:250:6]Wed Aug 9 18:00:02[2:242:5]Wed Aug 9 18:01:02 -
- - -
-
2 - Espionaje - (V) - 3[2:250:6]Wed Aug 9 17:59:55[2:242:1]Wed Aug 9 18:01:55 -
- - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Nueva misin: elegir naves
NavesDisponibles--
Nave pequea de carga10mx
Nave grande de carga19mx
Crucero6mx
Reciclador1mx
Sonda de espionaje139mx
Ninguna naveTodas las naves
- -

-
- - From 0cb0c238d520a8718e313b52cffc356a5a7561bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 22 Aug 2023 19:53:15 +0200 Subject: [PATCH 255/598] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108315) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers’ TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] --- Lib/ssl.py | 31 ++- Lib/test/test_ssl.py | 211 ++++++++++++++++++ ...-08-22-17-39-12.gh-issue-108310.fVM3sg.rst | 7 + 3 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst diff --git a/Lib/ssl.py b/Lib/ssl.py index 1d5873726441e4..ff363c75e7dfd8 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -975,7 +975,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -994,9 +994,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 6117ca3fdba1b7..ad5377ec059c0b 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -10,11 +10,14 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +import re import socket import select +import struct import time import enum import gc +import http.client import os import errno import pprint @@ -4659,6 +4662,214 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + self.enterContext(server) # starts it & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + class TestEnumerations(unittest.TestCase): def test_tlsversion(self): diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 00000000000000..403c77a9d480ee --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From 615f6e946db6d7eae66b8de9a75a6c58a84516a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 20:17:25 +0200 Subject: [PATCH 256/598] gh-106320: Remove _PyDict_GetItemStringWithError() function (#108313) Remove private _PyDict_GetItemStringWithError() function of the public C API: the new PyDict_GetItemStringRef() can be used instead. * Move private _PyDict_GetItemStringWithError() to the internal C API. * _testcapi get_code_extra_index() uses PyDict_GetItemStringRef(). Avoid using private functions in _testcapi which tests the public C API. --- Include/cpython/dictobject.h | 1 - Include/internal/pycore_dict.h | 1 + Modules/_testcapi/code.c | 9 +++++---- Objects/structseq.c | 1 + Python/codecs.c | 1 + Python/import.c | 3 ++- Python/initconfig.c | 1 + Python/pythonrun.c | 1 + 8 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 2a42794fdf0c85..470f59436748fd 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -36,7 +36,6 @@ PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key); -PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index df7bc7e58f6c97..8e9c27e3b294b8 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -13,6 +13,7 @@ extern "C" { // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); +extern PyObject* _PyDict_GetItemStringWithError(PyObject *, const char *); extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); diff --git a/Modules/_testcapi/code.c b/Modules/_testcapi/code.c index cadaf5eb94692e..691dd5fe043811 100644 --- a/Modules/_testcapi/code.c +++ b/Modules/_testcapi/code.c @@ -9,12 +9,12 @@ get_code_extra_index(PyInterpreterState* interp) { PyObject *interp_dict = PyInterpreterState_GetDict(interp); // borrowed assert(interp_dict); // real users would handle missing dict... somehow - PyObject *index_obj = _PyDict_GetItemStringWithError(interp_dict, key); // borrowed + PyObject *index_obj; + if (PyDict_GetItemStringRef(interp_dict, key, &index_obj) < 0) { + goto finally; + } Py_ssize_t index = 0; if (!index_obj) { - if (PyErr_Occurred()) { - goto finally; - } index = PyUnstable_Eval_RequestCodeExtraIndex(NULL); if (index < 0 || PyErr_Occurred()) { goto finally; @@ -31,6 +31,7 @@ get_code_extra_index(PyInterpreterState* interp) { } else { index = PyLong_AsSsize_t(index_obj); + Py_DECREF(index_obj); if (index == -1 && PyErr_Occurred()) { goto finally; } diff --git a/Objects/structseq.c b/Objects/structseq.c index 700f67c09c9e57..95c4c15710d169 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -8,6 +8,7 @@ */ #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_object.h" // _PyObject_GC_TRACK() diff --git a/Python/codecs.c b/Python/codecs.c index 4e47ff93a3691b..3c418512e3aa0b 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -10,6 +10,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/import.c b/Python/import.c index 56b2dc1a4ada2c..4abb2f6a06b48c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,7 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" - +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -15,6 +15,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_weakref.h" // _PyWeakref_GET_REF() + #include "marshal.h" // PyMarshal_ReadObjectFromString() #include "importdl.h" // _PyImport_DynLoadFiletab #include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED() diff --git a/Python/initconfig.c b/Python/initconfig.c index 787e583262406c..c017abeb90563e 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_fileutils.h" // _Py_HasFileSystemDefaultEncodeErrors #include "pycore_getopt.h" // _PyOS_GetOpt() #include "pycore_initconfig.h" // _PyStatus_OK() diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b2e04cfa317c00..a3de7792cf5bc5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -15,6 +15,7 @@ #include "pycore_ast.h" // PyAST_mod2obj #include "pycore_ceval.h" // _Py_EnterRecursiveCall #include "pycore_compile.h" // _PyAST_Compile() +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_object.h" // _PyDebug_PrintTotalRefs() #include "pycore_parser.h" // _PyParser_ASTFromString() From c1c9cd61da1c5b6318da9f7055d45ec82a9949ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 20:25:06 +0200 Subject: [PATCH 257/598] gh-89745: Remove commented code in getpath.c (#108307) --- Modules/getpath.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index 76e3c7e65249f4..fb1656ada01496 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -922,23 +922,6 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) } Py_DECREF(r); -#if 0 - PyObject *it = PyObject_GetIter(configDict); - for (PyObject *k = PyIter_Next(it); k; k = PyIter_Next(it)) { - if (!strcmp("__builtins__", PyUnicode_AsUTF8(k))) { - Py_DECREF(k); - continue; - } - fprintf(stderr, "%s = ", PyUnicode_AsUTF8(k)); - PyObject *o = PyDict_GetItem(configDict, k); - o = PyObject_Repr(o); - fprintf(stderr, "%s\n", PyUnicode_AsUTF8(o)); - Py_DECREF(o); - Py_DECREF(k); - } - Py_DECREF(it); -#endif - if (_PyConfig_FromDict(config, configDict) < 0) { _PyErr_WriteUnraisableMsg("reading getpath results", NULL); Py_DECREF(dict); @@ -949,4 +932,3 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) return _PyStatus_OK(); } - From 21dda09600848ac280481f7c64f8d9516dc69bb2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 20:30:18 +0200 Subject: [PATCH 258/598] gh-108303: Add Lib/test/test_cppext/ sub-directory (#108325) * Move test_cppext to its own directory * Rename setup_testcppext.py to setup.py * Rename _testcppext.cpp to extension.cpp * The source (extension.cpp) is now also copied by the test. --- Lib/test/{test_cppext.py => test_cppext/__init__.py} | 8 ++++---- Lib/test/{_testcppext.cpp => test_cppext/extension.cpp} | 2 ++ Lib/test/{setup_testcppext.py => test_cppext/setup.py} | 3 +-- Makefile.pre.in | 1 + 4 files changed, 8 insertions(+), 6 deletions(-) rename Lib/test/{test_cppext.py => test_cppext/__init__.py} (91%) rename Lib/test/{_testcppext.cpp => test_cppext/extension.cpp} (99%) rename Lib/test/{setup_testcppext.py => test_cppext/setup.py} (93%) diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext/__init__.py similarity index 91% rename from Lib/test/test_cppext.py rename to Lib/test/test_cppext/__init__.py index 7a143f44640137..8adff91c38efcf 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext/__init__.py @@ -10,9 +10,8 @@ MS_WINDOWS = (sys.platform == 'win32') - - -SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py') +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.cpp') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @support.requires_subprocess() @@ -41,7 +40,8 @@ def check_build(self, std_cpp03, extension_name): def _check_build(self, std_cpp03, extension_name, python_exe): pkg_dir = 'pkg' os.mkdir(pkg_dir) - shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) def run_cmd(operation, cmd): env = os.environ.copy() diff --git a/Lib/test/_testcppext.cpp b/Lib/test/test_cppext/extension.cpp similarity index 99% rename from Lib/test/_testcppext.cpp rename to Lib/test/test_cppext/extension.cpp index 82b471312dd2b9..90669b10cb2c6d 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -1,5 +1,7 @@ // gh-91321: Very basic C++ test extension to check that the Python C API is // compatible with C++ and does not emit C++ compiler warnings. +// +// The code is only built, not executed. // Always enable assertions #undef NDEBUG diff --git a/Lib/test/setup_testcppext.py b/Lib/test/test_cppext/setup.py similarity index 93% rename from Lib/test/setup_testcppext.py rename to Lib/test/test_cppext/setup.py index 22fe750085fd70..6867094a420436 100644 --- a/Lib/test/setup_testcppext.py +++ b/Lib/test/test_cppext/setup.py @@ -2,7 +2,6 @@ # compatible with C++ and does not emit C++ compiler warnings. import os import sys -from test import support from setuptools import setup, Extension @@ -10,7 +9,7 @@ MS_WINDOWS = (sys.platform == 'win32') -SOURCE = support.findfile('_testcppext.cpp') +SOURCE = 'extension.cpp' if not MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ diff --git a/Makefile.pre.in b/Makefile.pre.in index beccab45a235a5..d66764e6165409 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2156,6 +2156,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/support/_hypothesis_stubs \ test/test_asyncio \ test/test_capi \ + test/test_cppext \ test/test_ctypes \ test/test_email \ test/test_email/data \ From 35cb1605d08a77f1c18bd476b26391acaaa35599 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 22 Aug 2023 21:38:38 +0300 Subject: [PATCH 259/598] Docs: Add link to skip to datetime's format codes (#108027) --- Doc/library/datetime.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index bed19ad145a20c..400c369a9bb736 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -19,6 +19,10 @@ The :mod:`datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. +.. tip:: + + Skip to :ref:`the format codes `. + .. seealso:: Module :mod:`calendar` @@ -2322,6 +2326,8 @@ versus :meth:`strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ + .. _format-codes: + :meth:`strftime` and :meth:`strptime` Format Codes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 39de79b345f925ce3bbb79b33534872fe0c90877 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 22 Aug 2023 21:40:46 +0300 Subject: [PATCH 260/598] Document 3.13, 3.14 and future removals (#108055) --- Doc/whatsnew/3.12.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ff651bcd047289..b557e6cde6a209 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1112,6 +1112,10 @@ Modules (see :pep:`594`): * :mod:`!uu` * :mod:`!xdrlib` +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + APIs: * :class:`!configparser.LegacyInterpolation` (:gh:`90765`) @@ -1146,6 +1150,10 @@ Pending Removal in Python 3.14 Use :class:`ast.Constant` instead. (Contributed by Serhiy Storchaka in :gh:`90953`.) +* :mod:`asyncio`: the *msg* parameter of both + :meth:`asyncio.Future.cancel` and + :meth:`asyncio.Task.cancel` (:gh:`90985`) + * :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. @@ -1212,12 +1220,17 @@ Pending Removal in Python 3.14 May be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) +* Creating :c:data:`immutable types ` with mutable + bases using the C API (:gh:`95388`) + Pending Removal in Future Versions ---------------------------------- The following APIs were deprecated in earlier Python versions and will be removed, although there is currently no date scheduled for their removal. +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + * :class:`typing.Text` (:gh:`92332`) * Currently Python accepts numeric literals immediately followed by keywords, From 13966da71b693b1fae1a8ef66e34e2f0a90ec6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hadh=C3=A1zy=20Tam=C3=A1s?= <85063808+Hels15@users.noreply.github.com> Date: Tue, 22 Aug 2023 19:41:56 +0100 Subject: [PATCH 261/598] gh-105857: Document that asyncio subprocess std{in,out,err} can be file handles (#107986) stdin/out can be filehandles -> add to docs. --- Doc/library/asyncio-eventloop.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8f2d8f336c82bb..04af53b980ff9e 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1442,6 +1442,7 @@ async/await code consider using the high-level * *stdin* can be any of these: * a file-like object + * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()` * the :const:`subprocess.PIPE` constant (default) which will create a new pipe and connect it, * the value ``None`` which will make the subprocess inherit the file From e97b7bef4fbe71821d59d2f41f311e514fd29e39 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 22 Aug 2023 19:42:51 +0100 Subject: [PATCH 262/598] GH-92584: Remove distutils from the newtypes tutorial includes (#108024) --- Doc/extending/newtypes_tutorial.rst | 54 +++++++++++++-------------- Doc/includes/{ => newtypes}/custom.c | 0 Doc/includes/{ => newtypes}/custom2.c | 0 Doc/includes/{ => newtypes}/custom3.c | 0 Doc/includes/{ => newtypes}/custom4.c | 0 Doc/includes/newtypes/pyproject.toml | 7 ++++ Doc/includes/newtypes/setup.py | 8 ++++ Doc/includes/{ => newtypes}/sublist.c | 0 Doc/includes/{ => newtypes}/test.py | 7 ---- Doc/includes/setup.py | 9 ----- 10 files changed, 42 insertions(+), 43 deletions(-) rename Doc/includes/{ => newtypes}/custom.c (100%) rename Doc/includes/{ => newtypes}/custom2.c (100%) rename Doc/includes/{ => newtypes}/custom3.c (100%) rename Doc/includes/{ => newtypes}/custom4.c (100%) create mode 100644 Doc/includes/newtypes/pyproject.toml create mode 100644 Doc/includes/newtypes/setup.py rename Doc/includes/{ => newtypes}/sublist.c (100%) rename Doc/includes/{ => newtypes}/test.py (94%) delete mode 100644 Doc/includes/setup.py diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 209a4ab76d226d..7eba9759119b3b 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -45,7 +45,7 @@ extension module :mod:`!custom`: allows defining heap-allocated extension types using the :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. -.. literalinclude:: ../includes/custom.c +.. literalinclude:: ../includes/newtypes/custom.c Now that's quite a bit to take in at once, but hopefully bits will seem familiar from the previous chapter. This file defines three things: @@ -194,36 +194,32 @@ This adds the type to the module dictionary. This allows us to create >>> mycustom = custom.Custom() That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: +:file:`custom.c`, + +.. literalinclude:: ../includes/newtypes/pyproject.toml + +in a file called :file:`pyproject.toml`, and .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) + from setuptools import Extension, setup + setup(ext_modules=[Extension("custom", ["custom.c"])]) in a file called :file:`setup.py`; then typing .. code-block:: shell-session - $ python setup.py build + $ python -m pip install . -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. +in a shell should produce a file :file:`custom.so` in a subdirectory +and install it; now fire up Python --- you should be able to ``import custom`` +and play around with ``Custom`` objects. That wasn't so hard, was it? Of course, the current Custom type is pretty uninteresting. It has no data and doesn't do anything. It can't even be subclassed. -.. note:: - While this documentation showcases the standard :mod:`!distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide `_. - Adding data and methods to the Basic example ============================================ @@ -232,7 +228,7 @@ Let's extend the basic example to add some data and methods. Let's also make the type usable as a base class. We'll create a new module, :mod:`!custom2` that adds these capabilities: -.. literalinclude:: ../includes/custom2.c +.. literalinclude:: ../includes/newtypes/custom2.c This version of the module has a number of changes. @@ -514,17 +510,21 @@ We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the module name in the :c:type:`PyModuleDef` struct, and update the full class name in the :c:type:`PyTypeObject` struct. -Finally, we update our :file:`setup.py` file to build the new module: +Finally, we update our :file:`setup.py` file to include the new module, .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) + from setuptools import Extension, setup + setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + ]) + +and then we re-install so that we can ``import custom2``: + +.. code-block:: shell-session + $ python -m pip install . Providing finer control over data attributes ============================================ @@ -535,7 +535,7 @@ version of our module, the instance variables :attr:`!first` and :attr:`!last` could be set to non-string values or even deleted. We want to make sure that these attributes always contain strings. -.. literalinclude:: ../includes/custom3.c +.. literalinclude:: ../includes/newtypes/custom3.c To provide greater control, over the :attr:`!first` and :attr:`!last` attributes, @@ -682,7 +682,7 @@ To allow a :class:`!Custom` instance participating in a reference cycle to be properly detected and collected by the cyclic GC, our :class:`!Custom` type needs to fill two additional slots and to enable a flag that enables these slots: -.. literalinclude:: ../includes/custom4.c +.. literalinclude:: ../includes/newtypes/custom4.c First, the traversal method lets the cyclic GC know about subobjects that could @@ -806,7 +806,7 @@ increases an internal counter: >>> print(s.increment()) 2 -.. literalinclude:: ../includes/sublist.c +.. literalinclude:: ../includes/newtypes/sublist.c As you can see, the source code closely resembles the :class:`!Custom` examples in diff --git a/Doc/includes/custom.c b/Doc/includes/newtypes/custom.c similarity index 100% rename from Doc/includes/custom.c rename to Doc/includes/newtypes/custom.c diff --git a/Doc/includes/custom2.c b/Doc/includes/newtypes/custom2.c similarity index 100% rename from Doc/includes/custom2.c rename to Doc/includes/newtypes/custom2.c diff --git a/Doc/includes/custom3.c b/Doc/includes/newtypes/custom3.c similarity index 100% rename from Doc/includes/custom3.c rename to Doc/includes/newtypes/custom3.c diff --git a/Doc/includes/custom4.c b/Doc/includes/newtypes/custom4.c similarity index 100% rename from Doc/includes/custom4.c rename to Doc/includes/newtypes/custom4.c diff --git a/Doc/includes/newtypes/pyproject.toml b/Doc/includes/newtypes/pyproject.toml new file mode 100644 index 00000000000000..ea7937a3171473 --- /dev/null +++ b/Doc/includes/newtypes/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "custom" +version = "1" diff --git a/Doc/includes/newtypes/setup.py b/Doc/includes/newtypes/setup.py new file mode 100644 index 00000000000000..67f83673bcc657 --- /dev/null +++ b/Doc/includes/newtypes/setup.py @@ -0,0 +1,8 @@ +from setuptools import Extension, setup +setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + Extension("custom3", ["custom3.c"]), + Extension("custom4", ["custom4.c"]), + Extension("sublist", ["sublist.c"]), +]) diff --git a/Doc/includes/sublist.c b/Doc/includes/newtypes/sublist.c similarity index 100% rename from Doc/includes/sublist.c rename to Doc/includes/newtypes/sublist.c diff --git a/Doc/includes/test.py b/Doc/includes/newtypes/test.py similarity index 94% rename from Doc/includes/test.py rename to Doc/includes/newtypes/test.py index 09ebe3fec0bdbe..55a5cf9f68b94a 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/newtypes/test.py @@ -187,13 +187,6 @@ >>> gc.enable() """ -import os -import sys -from distutils.util import get_platform -PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) -src = os.path.join("build", "lib.%s" % PLAT_SPEC) -sys.path.append(src) - if __name__ == "__main__": import doctest, __main__ doctest.testmod(__main__) diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py deleted file mode 100644 index a38a39de3e7c86..00000000000000 --- a/Doc/includes/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup, Extension -setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - Extension("noddy3", ["noddy3.c"]), - Extension("noddy4", ["noddy4.c"]), - Extension("shoddy", ["shoddy.c"]), - ]) From fc23f34cc9701949e6832eb32f26ea89f6622b82 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 22 Aug 2023 21:45:22 +0300 Subject: [PATCH 263/598] gh-107136: Remove Plausible for docs metrics (#107856) --- Doc/tools/templates/layout.html | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 80103158ea01e6..9498b2ccc5af92 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,9 +26,6 @@ {% endblock %} {% block extrahead %} - {% if builder == "html" %} - - {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From a0bb4a39d1ca10e4a75f50a9fbe90cc9db28d29e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Aug 2023 20:49:39 +0100 Subject: [PATCH 264/598] Fix spurious diff if the cases generator is run on Windows (#108319) --- Tools/cases_generator/generate_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 6ee68363058116..f605dcc5e4632f 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -237,7 +237,7 @@ def from_source_files(self) -> str: except ValueError: # May happen on Windows if root and temp on different volumes pass - filenames.append(filename) + filenames.append(filename.replace(os.path.sep, posixpath.sep)) paths = f"\n{self.out.comment} ".join(filenames) return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" From 3a1ac87f8f89d3206b46a0df4908afae629d669d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 01:39:50 +0200 Subject: [PATCH 265/598] gh-90791: test.pythoninfo logs ASAN_OPTIONS env var (#108289) * Cleanup libregrtest code logging ASAN_OPTIONS. * Fix a typo on "ASAN_OPTIONS" vs "MSAN_OPTIONS". --- Lib/test/libregrtest/main.py | 41 +++++++++++++++++++++--------------- Lib/test/pythoninfo.py | 7 ++++++ Lib/test/support/__init__.py | 16 +++++++------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9001ca33b6166a..361189199d3820 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -526,26 +526,33 @@ def display_header(self): print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + self.display_sanitizers() + + def display_sanitizers(self): + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) msan = support.check_sanitizer(memory=True) ubsan = support.check_sanitizer(ub=True) - # This makes it easier to remember what to set in your local - # environment when trying to reproduce a sanitizer failure. - if asan or msan or ubsan: - names = [n for n in (asan and "address", - msan and "memory", - ubsan and "undefined behavior") - if n] - print(f"== sanitizers: {', '.join(names)}") - a_opts = os.environ.get("ASAN_OPTIONS") - if asan and a_opts is not None: - print(f"== ASAN_OPTIONS={a_opts}") - m_opts = os.environ.get("ASAN_OPTIONS") - if msan and m_opts is not None: - print(f"== MSAN_OPTIONS={m_opts}") - ub_opts = os.environ.get("UBSAN_OPTIONS") - if ubsan and ub_opts is not None: - print(f"== UBSAN_OPTIONS={ub_opts}") + sanitizers = [] + if asan: + sanitizers.append("address") + if msan: + sanitizers.append("memory") + if ubsan: + sanitizers.append("undefined behavior") + if not sanitizers: + return + + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") def no_tests_run(self): return not any((self.good, self.bad, self.skipped, self.interrupted, diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index ad7d5291af42f3..53af21db0755b1 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -308,6 +308,13 @@ def format_groups(groups): "_PYTHON_PROJECT_BASE", "_PYTHON_SYSCONFIGDATA_NAME", "__PYVENV_LAUNCHER__", + + # Sanitizer options + "ASAN_OPTIONS", + "LSAN_OPTIONS", + "MSAN_OPTIONS", + "TSAN_OPTIONS", + "UBSAN_OPTIONS", )) for name, value in os.environ.items(): uname = name.upper() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 64c66d8e25d9cd..8e8bf49a5cb5ce 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -407,19 +407,19 @@ def check_sanitizer(*, address=False, memory=False, ub=False): raise ValueError('At least one of address, memory, or ub must be True') - _cflags = sysconfig.get_config_var('CFLAGS') or '' - _config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('CFLAGS') or '' + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' memory_sanitizer = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args + '-fsanitize=memory' in cflags or + '--with-memory-sanitizer' in config_args ) address_sanitizer = ( - '-fsanitize=address' in _cflags or - '--with-address-sanitizer' in _config_args + '-fsanitize=address' in cflags or + '--with-address-sanitizer' in config_args ) ub_sanitizer = ( - '-fsanitize=undefined' in _cflags or - '--with-undefined-behavior-sanitizer' in _config_args + '-fsanitize=undefined' in cflags or + '--with-undefined-behavior-sanitizer' in config_args ) return ( (memory and memory_sanitizer) or From 9173b2bbe13aeccc075b571da05c653a2a91de1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 04:52:56 +0200 Subject: [PATCH 266/598] gh-105776: Fix test_cppext when CC contains -std=c11 option (#108343) Fix test_cppext when the C compiler command has the "-std=c11" option. Remove "-std=" options from the compiler command. --- Lib/test/test_cppext/setup.py | 13 +++++++++++++ .../2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index 6867094a420436..976633bc33889c 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,7 +1,9 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import shlex import sys +import sysconfig from setuptools import setup, Extension @@ -30,6 +32,17 @@ def main(): cppflags = [*CPPFLAGS, f'-std={std}'] + # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 + # option emits a C++ compiler warning. Remove "-std11" option from the + # CC command. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + cpp_ext = Extension( name, sources=[SOURCE], diff --git a/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst new file mode 100644 index 00000000000000..0e0a3aa9b11e68 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst @@ -0,0 +1,2 @@ +Fix test_cppext when the C compiler command ``-std=c11`` option: remove +``-std=`` options from the compiler command. Patch by Victor Stinner. From 64f99350351bc46e016b2286f36ba7cd669b79e3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 07:26:01 +0200 Subject: [PATCH 267/598] gh-108342: Break ref cycle in SSLSocket._create() exc (#108344) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of #108310. --- Lib/ssl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index ff363c75e7dfd8..c4c5a4ca894ee5 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1021,7 +1021,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From 79fdacc0059a3959074d2d9d054653eae1dcfe06 Mon Sep 17 00:00:00 2001 From: FrozenBob <30644137+FrozenBob@users.noreply.github.com> Date: Wed, 23 Aug 2023 00:11:15 -0700 Subject: [PATCH 268/598] gh-108267: Dataclasses docs: Fix object.__setattr__ typo (#108355) Fixed a sentence in dataclasses.rst Changed "__setattr__" to "object.__setattr__" in a section that was specifically supposed to refer to the __setattr__ method of the object class. Also suppressed the link to the data model docs for __setattr__, since we're talking about a specific __setattr__ implementation, not __setattr__ methods in general. --- Doc/library/dataclasses.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 535a60ccca8d07..d68748767c5e37 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -609,7 +609,7 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`~object.__setattr__`. +must use :meth:`!object.__setattr__`. Inheritance ----------- From 2dfbd4f36dd83f88f5df64c33612dd34eff256bb Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 23 Aug 2023 09:01:17 +0100 Subject: [PATCH 269/598] gh-108113: Make it possible to optimize an AST (#108282) --- Include/internal/pycore_compile.h | 8 +++++++ Lib/test/test_ast.py | 26 ++++++++++++--------- Lib/test/test_builtin.py | 12 ++++++---- Python/bltinmodule.c | 39 ++++++++++++++++++++++--------- Python/compile.c | 18 ++++++++++++++ Python/pythonrun.c | 22 +---------------- 6 files changed, 77 insertions(+), 48 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index ad657c0f0fcedc..0167e590e63fb1 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -19,6 +19,14 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( int optimize, struct _arena *arena); +/* AST optimizations */ +PyAPI_FUNC(int) _PyCompile_AstOptimize( + struct _mod *mod, + PyObject *filename, + PyCompilerFlags *flags, + int optimize, + struct _arena *arena); + static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; extern int _PyAST_Optimize( diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f3c7229f0b6c76..68de4d6f3f1923 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -361,14 +361,16 @@ def test_optimization_levels__debug__(self): cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] for (optval, expected) in cases: with self.subTest(optval=optval, expected=expected): - res = ast.parse("__debug__", optimize=optval) - self.assertIsInstance(res.body[0], ast.Expr) - if isinstance(expected, bool): - self.assertIsInstance(res.body[0].value, ast.Constant) - self.assertEqual(res.body[0].value.value, expected) - else: - self.assertIsInstance(res.body[0].value, ast.Name) - self.assertEqual(res.body[0].value.id, expected) + res1 = ast.parse("__debug__", optimize=optval) + res2 = ast.parse(ast.parse("__debug__"), optimize=optval) + for res in [res1, res2]: + self.assertIsInstance(res.body[0], ast.Expr) + if isinstance(expected, bool): + self.assertIsInstance(res.body[0].value, ast.Constant) + self.assertEqual(res.body[0].value.value, expected) + else: + self.assertIsInstance(res.body[0].value, ast.Name) + self.assertEqual(res.body[0].value.id, expected) def test_optimization_levels_const_folding(self): folded = ('Expr', (1, 0, 1, 5), ('Constant', (1, 0, 1, 5), 3, None)) @@ -381,9 +383,11 @@ def test_optimization_levels_const_folding(self): cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] for (optval, expected) in cases: with self.subTest(optval=optval): - tree = ast.parse("1 + 2", optimize=optval) - res = to_tuple(tree.body[0]) - self.assertEqual(res, expected) + tree1 = ast.parse("1 + 2", optimize=optval) + tree2 = ast.parse(ast.parse("1 + 2"), optimize=optval) + for tree in [tree1, tree2]: + res = to_tuple(tree.body[0]) + self.assertEqual(res, expected) def test_invalid_position_information(self): invalid_linenos = [ diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index ee3ba6ab07bbdf..dbc706ae7b41f5 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -521,9 +521,10 @@ def test_compile_async_generator(self): def test_compile_ast(self): args = ("a*(1+2)", "f.py", "exec") raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] - opt = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0] - for tree in (raw, opt): + for tree in (raw, opt1, opt2): self.assertIsInstance(tree.value, ast.BinOp) self.assertIsInstance(tree.value.op, ast.Mult) self.assertIsInstance(tree.value.left, ast.Name) @@ -536,9 +537,10 @@ def test_compile_ast(self): self.assertIsInstance(raw_right.right, ast.Constant) self.assertEqual(raw_right.right.value, 2) - opt_right = opt.value.right # expect Constant(3) - self.assertIsInstance(opt_right, ast.Constant) - self.assertEqual(opt_right.value, 3) + for opt in [opt1, opt2]: + opt_right = opt.value.right # expect Constant(3) + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, 3) def test_delattr(self): sys.spam = 1 diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d06efcf6ba3687..787f53fae354db 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -804,23 +804,40 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (is_ast == -1) goto error; if (is_ast) { - if (flags & PyCF_ONLY_AST) { + if ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST) { + // return an un-optimized AST result = Py_NewRef(source); } else { - PyArena *arena; - mod_ty mod; + // Return an optimized AST or code object - arena = _PyArena_New(); - if (arena == NULL) - goto error; - mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); + PyArena *arena = _PyArena_New(); + if (arena == NULL) { goto error; } - result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + + if (flags & PyCF_ONLY_AST) { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + goto error; + } + if (_PyCompile_AstOptimize(mod, filename, &cf, optimize, + arena) < 0) { + _PyArena_Free(arena); + goto error; + } + result = PyAST_mod2obj(mod); + } + else { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + goto error; + } + result = (PyObject*)_PyAST_Compile(mod, filename, + &cf, optimize, arena); + } _PyArena_Free(arena); } goto finally; diff --git a/Python/compile.c b/Python/compile.c index 3260dba57eac8f..4b2f70a7ef01d7 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -557,6 +557,24 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, return co; } +int +_PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, + int optimize, PyArena *arena) +{ + PyFutureFeatures future; + if (!_PyFuture_FromAST(mod, filename, &future)) { + return -1; + } + int flags = future.ff_features | cf->cf_flags; + if (optimize == -1) { + optimize = _Py_GetConfig()->optimization_level; + } + if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + return -1; + } + return 0; +} + static void compiler_free(struct compiler *c) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a3de7792cf5bc5..05b7dfa61d1a52 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -22,7 +22,6 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_symtable.h" // _PyFuture_FromAST() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() @@ -1792,24 +1791,6 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, return NULL; } -static int -ast_optimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena) -{ - PyFutureFeatures future; - if (!_PyFuture_FromAST(mod, filename, &future)) { - return -1; - } - int flags = future.ff_features | cf->cf_flags; - if (optimize == -1) { - optimize = _Py_GetConfig()->optimization_level; - } - if (!_PyAST_Optimize(mod, arena, optimize, flags)) { - return -1; - } - return 0; -} - PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) @@ -1827,8 +1808,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { - if (ast_optimize(mod, filename, flags, optimize, arena) < 0) { - _PyArena_Free(arena); + if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena) < 0) { return NULL; } } From 31b61d19abcc63aa28625a31ed75411948fc1e7e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 23 Aug 2023 11:00:22 +0200 Subject: [PATCH 270/598] gh-108294: Add time.sleep audit event (GH-108298) --- Doc/library/time.rst | 5 +++++ Lib/test/audit-tests.py | 15 +++++++++++++++ Lib/test/test_audit.py | 15 +++++++++++++++ ...2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst | 1 + Modules/timemodule.c | 2 ++ 5 files changed, 38 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 9f23a6fc7d5341..6ffe4ac4284140 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -379,6 +379,8 @@ Functions * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); * Or use ``select()`` (resolution: 1 microsecond). + .. audit-event:: time.sleep secs + .. versionchanged:: 3.11 On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now used if available. On Windows, a waitable timer is now used. @@ -389,6 +391,9 @@ Functions :pep:`475` for the rationale). + .. versionchanged:: 3.13 + Raises an auditing event. + .. index:: single: % (percent); datetime format diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 9504829e96f00e..cc614eab908500 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -514,6 +514,21 @@ def test_not_in_gc(): assert hook not in o +def test_time(): + import time + + def hook(event, args): + if event.startswith("time."): + print(event, *args) + sys.addaudithook(hook) + + time.sleep(0) + time.sleep(0.0625) # 1/16, a small exact float + try: + time.sleep(-1) + except ValueError: + pass + def test_sys_monitoring_register_callback(): import sys diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index b12ffa5d872e83..3a15835917cc32 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -256,6 +256,21 @@ def test_not_in_gc(self): if returncode: self.fail(stderr) + def test_time(self): + returncode, events, stderr = self.run_python("test_time") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + + actual = [(ev[0], ev[2]) for ev in events] + expected = [("time.sleep", "0"), + ("time.sleep", "0.0625"), + ("time.sleep", "-1")] + + self.assertEqual(actual, expected) + def test_sys_monitoring_register_callback(self): returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") diff --git a/Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst b/Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst new file mode 100644 index 00000000000000..de2a3a8a8ad891 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst @@ -0,0 +1 @@ +:func:`time.sleep` now raises an auditing event. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 912710219bd014..68948b6be1a61a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -414,6 +414,8 @@ Return the clk_id of a thread's CPU time clock."); static PyObject * time_sleep(PyObject *self, PyObject *timeout_obj) { + PySys_Audit("time.sleep", "O", timeout_obj); + _PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) return NULL; From 29bc6165ab8aa434145a34676b8b7e48e7c6e308 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 23 Aug 2023 12:10:08 +0200 Subject: [PATCH 271/598] gh-105539: Fix ResourceWarning from unclosed SQLite connections in test_sqlite3 (#108360) Follow up to 1a1bfc28912a39b500c578e9f10a8a222638d411. Explicitly manage connections in: - test_audit.test_sqlite3 - test_sqlite3.test_audit - test_sqlite3.test_backup Co-authored-by: Erlend E. Aasland --- Lib/test/audit-tests.py | 21 ++++++++++++--------- Lib/test/test_sqlite3/test_backup.py | 2 +- Lib/test/test_sqlite3/test_dbapi.py | 6 +++--- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index cc614eab908500..ad8f72f556331d 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -398,15 +398,18 @@ def hook(event, *args): cx2 = sqlite3.Connection(":memory:") # Configured without --enable-loadable-sqlite-extensions - if hasattr(sqlite3.Connection, "enable_load_extension"): - cx1.enable_load_extension(False) - try: - cx1.load_extension("test") - except sqlite3.OperationalError: - pass - else: - raise RuntimeError("Expected sqlite3.load_extension to fail") - + try: + if hasattr(sqlite3.Connection, "enable_load_extension"): + cx1.enable_load_extension(False) + try: + cx1.load_extension("test") + except sqlite3.OperationalError: + pass + else: + raise RuntimeError("Expected sqlite3.load_extension to fail") + finally: + cx1.close() + cx2.close() def test_sys_getframe(): import sys diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index 4584d976bce0c6..c7400d8b2165e6 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -137,7 +137,7 @@ def progress(status, remaining, total): raise SystemError('nearly out of space') with self.assertRaises(SystemError) as err: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.assertEqual(str(err.exception), 'nearly out of space') diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index d80ad7af3200f0..9dedbdbc4bb6d2 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -40,6 +40,7 @@ from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin class ModuleTests(unittest.TestCase): @@ -1740,10 +1741,9 @@ def test_closed_call(self): self.check(self.con) -class ClosedCurTests(unittest.TestCase): +class ClosedCurTests(MemoryDatabaseMixin, unittest.TestCase): def test_closed(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cx.cursor() cur.close() for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"): From 154477be722ae5c4e18d22d0860e284006b09c4f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Aug 2023 15:23:41 +0300 Subject: [PATCH 272/598] gh-50002: xml.dom.minidom now preserves whitespaces in attributes (GH-107947) Also double quotes (") are now only quoted in attributes. --- Lib/test/test_minidom.py | 40 +++++++++++++++++++ Lib/xml/dom/minidom.py | 30 ++++++++++---- ...3-08-14-20-01-14.gh-issue-50002.E-bpj8.rst | 1 + ...3-08-14-20-18-59.gh-issue-81555.cWdP4a.rst | 1 + 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst create mode 100644 Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 699265ccadc7f9..3ecd1af31eea77 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -505,6 +505,46 @@ def testWriteXML(self): dom.unlink() self.confirm(str == domstr) + def test_toxml_quote_text(self): + dom = Document() + elem = dom.appendChild(dom.createElement('elem')) + elem.appendChild(dom.createTextNode('&<>"')) + cr = elem.appendChild(dom.createElement('cr')) + cr.appendChild(dom.createTextNode('\r')) + crlf = elem.appendChild(dom.createElement('crlf')) + crlf.appendChild(dom.createTextNode('\r\n')) + lflf = elem.appendChild(dom.createElement('lflf')) + lflf.appendChild(dom.createTextNode('\n\n')) + ws = elem.appendChild(dom.createElement('ws')) + ws.appendChild(dom.createTextNode('\t\n\r ')) + domstr = dom.toxml() + dom.unlink() + self.assertEqual(domstr, '' + '&<>"' + '\r' + '\r\n' + '\n\n' + '\t\n\r ') + + def test_toxml_quote_attrib(self): + dom = Document() + elem = dom.appendChild(dom.createElement('elem')) + elem.setAttribute("a", '&<>"') + elem.setAttribute("cr", "\r") + elem.setAttribute("lf", "\n") + elem.setAttribute("crlf", "\r\n") + elem.setAttribute("lflf", "\n\n") + elem.setAttribute("ws", "\t\n\r ") + domstr = dom.toxml() + dom.unlink() + self.assertEqual(domstr, '' + '') + def testAltNewline(self): str = '\n
\n' dom = parseString(str) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index ef8a159833bbc0..db51f350ea0153 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -300,12 +300,28 @@ def _in_document(node): node = node.parentNode return False -def _write_data(writer, data): +def _write_data(writer, text, attr): "Writes datachars to writer." - if data: - data = data.replace("&", "&").replace("<", "<"). \ - replace("\"", """).replace(">", ">") - writer.write(data) + if not text: + return + # See the comments in ElementTree.py for behavior and + # implementation details. + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + if attr: + if '"' in text: + text = text.replace('"', """) + if "\r" in text: + text = text.replace("\r", " ") + if "\n" in text: + text = text.replace("\n", " ") + if "\t" in text: + text = text.replace("\t", " ") + writer.write(text) def _get_elements_by_tagName_helper(parent, name, rc): for node in parent.childNodes: @@ -883,7 +899,7 @@ def writexml(self, writer, indent="", addindent="", newl=""): for a_name in attrs.keys(): writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value) + _write_data(writer, attrs[a_name].value, True) writer.write("\"") if self.childNodes: writer.write(">") @@ -1112,7 +1128,7 @@ def splitText(self, offset): return newText def writexml(self, writer, indent="", addindent="", newl=""): - _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + _write_data(writer, "%s%s%s" % (indent, self.data, newl), False) # DOM Level 3 (WD 9 April 2002) diff --git a/Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst b/Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst new file mode 100644 index 00000000000000..ca5c0740802eae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst @@ -0,0 +1 @@ +:mod:`xml.dom.minidom` now preserves whitespaces in attributes. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst b/Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst new file mode 100644 index 00000000000000..241a50f8b41c2b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst @@ -0,0 +1 @@ +:mod:`xml.dom.minidom` now only quotes ``"`` in attributes. From f5559f38d9831e7e55a518e516bcd620ec13af14 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 17:40:26 +0200 Subject: [PATCH 273/598] gh-108308: Replace PyDict_GetItem() with PyDict_GetItemRef() (#108309) Replace PyDict_GetItem() calls with PyDict_GetItemRef() or PyDict_GetItemWithError() to handle errors. * Replace PyLong_AS_LONG() with _PyLong_AsInt() and check for errors. * Check for PyDict_Contains() error. * pycore_init_builtins() checks for _PyType_Lookup() failure. --- Python/assemble.c | 49 ++++++++++++++++++++++++++++++++++---------- Python/compile.c | 34 +++++++++++++++++++++++------- Python/flowgraph.c | 30 +++++++++++++++++++-------- Python/pylifecycle.c | 24 ++++++++++++++++------ 4 files changed, 105 insertions(+), 32 deletions(-) diff --git a/Python/assemble.c b/Python/assemble.c index b7012534d6cc4e..4f66cf294e38c9 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -18,7 +18,7 @@ #define ERROR -1 #define RETURN_IF_ERROR(X) \ - if ((X) == -1) { \ + if ((X) < 0) { \ return ERROR; \ } @@ -448,13 +448,17 @@ static PyObject * dict_keys_inorder(PyObject *dict, Py_ssize_t offset) { PyObject *tuple, *k, *v; - Py_ssize_t i, pos = 0, size = PyDict_GET_SIZE(dict); + Py_ssize_t pos = 0, size = PyDict_GET_SIZE(dict); tuple = PyTuple_New(size); if (tuple == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyLong_AS_LONG(v); + Py_ssize_t i = PyLong_AsSsize_t(v); + if (i == -1 && PyErr_Occurred()) { + Py_DECREF(tuple); + return NULL; + } assert((i - offset) < size); assert((i - offset) >= 0); PyTuple_SET_ITEM(tuple, i - offset, Py_NewRef(k)); @@ -466,24 +470,34 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); -static void +static int compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, PyObject *names, PyObject *kinds) { PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_varnames, &pos, &k, &v)) { - int offset = (int)PyLong_AS_LONG(v); + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); assert(offset < nlocalsplus); + // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_Contains(umd->u_fasthidden, k)) { + int has_key = PyDict_Contains(umd->u_fasthidden, k); + RETURN_IF_ERROR(has_key); + if (has_key) { kind |= CO_FAST_HIDDEN; } - if (PyDict_GetItem(umd->u_cellvars, k) != NULL) { + + has_key = PyDict_Contains(umd->u_cellvars, k); + RETURN_IF_ERROR(has_key); + if (has_key) { kind |= CO_FAST_CELL; } + _Py_set_localsplus_info(offset, k, kind, names, kinds); } int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); @@ -492,12 +506,18 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, int numdropped = 0; pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &k, &v)) { - if (PyDict_GetItem(umd->u_varnames, k) != NULL) { + int has_name = PyDict_Contains(umd->u_varnames, k); + RETURN_IF_ERROR(has_name); + if (has_name) { // Skip cells that are already covered by locals. numdropped += 1; continue; } - int offset = (int)PyLong_AS_LONG(v); + + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); offset += nlocals - numdropped; assert(offset < nlocalsplus); @@ -506,12 +526,16 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, pos = 0; while (PyDict_Next(umd->u_freevars, &pos, &k, &v)) { - int offset = (int)PyLong_AS_LONG(v); + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); offset += nlocals - numdropped; assert(offset < nlocalsplus); _Py_set_localsplus_info(offset, k, CO_FAST_FREE, names, kinds); } + return SUCCESS; } static PyCodeObject * @@ -556,7 +580,10 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ if (localspluskinds == NULL) { goto error; } - compute_localsplus_info(umd, nlocalsplus, localsplusnames, localspluskinds); + if (compute_localsplus_info(umd, nlocalsplus, + localsplusnames, localspluskinds) == ERROR) { + goto error; + } struct _PyCodeConstructor con = { .filename = filename, diff --git a/Python/compile.c b/Python/compile.c index 4b2f70a7ef01d7..b67a1885ddc983 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4212,9 +4212,20 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste) || - (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True)) + if (_PyST_IsFunctionLike(c->u->u_ste)) { optype = OP_FAST; + } + else { + PyObject *item; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, + &item) < 0) { + goto error; + } + if (item == Py_True) { + optype = OP_FAST; + } + Py_XDECREF(item); + } break; case GLOBAL_IMPLICIT: if (_PyST_IsFunctionLike(c->u->u_ste)) @@ -4239,7 +4250,7 @@ compiler_nameop(struct compiler *c, location loc, op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { - return ERROR; + goto error; } } else if (c->u->u_ste->ste_can_see_class_scope) { @@ -4247,7 +4258,7 @@ compiler_nameop(struct compiler *c, location loc, // First load the classdict if (compiler_addop_o(c->u, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - return ERROR; + goto error; } } else { @@ -4274,7 +4285,7 @@ compiler_nameop(struct compiler *c, location loc, // First load the classdict if (compiler_addop_o(c->u, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - return ERROR; + goto error; } } else { op = LOAD_GLOBAL; @@ -4308,6 +4319,10 @@ compiler_nameop(struct compiler *c, location loc, arg <<= 1; } return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); + +error: + Py_DECREF(mangled); + return ERROR; } static int @@ -5536,8 +5551,13 @@ push_inlined_comprehension_state(struct compiler *c, location loc, if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { if (!_PyST_IsFunctionLike(c->u->u_ste)) { // non-function scope: override this name to use fast locals - PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); - if (orig != Py_True) { + PyObject *orig; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { + return ERROR; + } + int orig_is_true = (orig == Py_True); + Py_XDECREF(orig); + if (!orig_is_true) { if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { return ERROR; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 719ed92105074e..e620e7cf1b9e96 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2404,17 +2404,31 @@ build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) PyObject *varname, *cellindex; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); - if (varindex != NULL) { - assert(PyLong_AS_LONG(cellindex) < INT_MAX); - assert(PyLong_AS_LONG(varindex) < INT_MAX); - int oldindex = (int)PyLong_AS_LONG(cellindex); - int argoffset = (int)PyLong_AS_LONG(varindex); - fixed[oldindex] = argoffset; + PyObject *varindex; + if (PyDict_GetItemRef(umd->u_varnames, varname, &varindex) < 0) { + goto error; + } + if (varindex == NULL) { + continue; + } + + int argoffset = _PyLong_AsInt(varindex); + Py_DECREF(varindex); + if (argoffset == -1 && PyErr_Occurred()) { + goto error; } - } + int oldindex = _PyLong_AsInt(cellindex); + if (oldindex == -1 && PyErr_Occurred()) { + goto error; + } + fixed[oldindex] = argoffset; + } return fixed; + +error: + PyMem_Free(fixed); + return NULL; } #define IS_GENERATOR(CF) \ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9837e0f0d52e6d..8c321fbcfe4104 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -762,18 +762,30 @@ pycore_init_builtins(PyThreadState *tstate) } interp->builtins = Py_NewRef(builtins_dict); - PyObject *isinstance = PyDict_GetItem(builtins_dict, &_Py_ID(isinstance)); - assert(isinstance); + PyObject *isinstance = PyDict_GetItemWithError(builtins_dict, &_Py_ID(isinstance)); + if (!isinstance) { + goto error; + } interp->callable_cache.isinstance = isinstance; - PyObject *len = PyDict_GetItem(builtins_dict, &_Py_ID(len)); - assert(len); + + PyObject *len = PyDict_GetItemWithError(builtins_dict, &_Py_ID(len)); + if (!len) { + goto error; + } interp->callable_cache.len = len; + PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); - assert(list_append); + if (list_append == NULL) { + goto error; + } interp->callable_cache.list_append = list_append; + PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); - assert(object__getattribute__); + if (object__getattribute__ == NULL) { + goto error; + } interp->callable_cache.object__getattribute__ = object__getattribute__; + if (_PyBuiltins_AddExceptions(bimod) < 0) { return _PyStatus_ERR("failed to add exceptions to builtins"); } From 3f61cf646d0506baa0c0c2118f05110446519c62 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 23 Aug 2023 18:42:08 +0300 Subject: [PATCH 274/598] gh-108303: Move `ann_module*.py` files to `typinganndata/` folder (#108354) --- Lib/test/test_grammar.py | 6 +++--- Lib/test/test_inspect.py | 2 +- Lib/test/test_module/__init__.py | 4 +++- Lib/test/test_opcodes.py | 3 ++- Lib/test/test_typing.py | 6 ++++-- Lib/test/{ => typinganndata}/ann_module.py | 0 Lib/test/{ => typinganndata}/ann_module2.py | 0 Lib/test/{ => typinganndata}/ann_module3.py | 0 Lib/test/{ => typinganndata}/ann_module4.py | 0 Lib/test/{ => typinganndata}/ann_module5.py | 0 Lib/test/{ => typinganndata}/ann_module6.py | 0 Lib/test/{ => typinganndata}/ann_module7.py | 0 Lib/test/{ => typinganndata}/ann_module8.py | 0 13 files changed, 13 insertions(+), 8 deletions(-) rename Lib/test/{ => typinganndata}/ann_module.py (100%) rename Lib/test/{ => typinganndata}/ann_module2.py (100%) rename Lib/test/{ => typinganndata}/ann_module3.py (100%) rename Lib/test/{ => typinganndata}/ann_module4.py (100%) rename Lib/test/{ => typinganndata}/ann_module5.py (100%) rename Lib/test/{ => typinganndata}/ann_module6.py (100%) rename Lib/test/{ => typinganndata}/ann_module7.py (100%) rename Lib/test/{ => typinganndata}/ann_module8.py (100%) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ee105a3de17f8a..8507a07e498532 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -12,9 +12,9 @@ # different import patterns to check that __annotations__ does not interfere # with import machinery -import test.ann_module as ann_module +import test.typinganndata.ann_module as ann_module import typing -from test import ann_module2 +from test.typinganndata import ann_module2 import test # These are shared with test_tokenize and other test modules. @@ -452,7 +452,7 @@ def test_var_annot_module_semantics(self): def test_var_annot_in_module(self): # check that functions fail the same way when executed # outside of module where they were defined - ann_module3 = import_helper.import_fresh_module("test.ann_module3") + ann_module3 = import_helper.import_fresh_module("test.typinganndata.ann_module3") with self.assertRaises(NameError): ann_module3.f_bad_ann() with self.assertRaises(NameError): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 07c48eac5b48b0..9cb92c02d3e7d8 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4726,7 +4726,7 @@ def func(*args, **kwargs): def test_base_class_have_text_signature(self): # see issue 43118 - from test.ann_module7 import BufferedReader + from test.typinganndata.ann_module7 import BufferedReader class MyBufferedReader(BufferedReader): """buffer reader class.""" diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index cfc4d9ccf1cc81..2524e6c87cb459 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -324,7 +324,9 @@ def test_annotations_getset_raises(self): del foo.__annotations__ def test_annotations_are_created_correctly(self): - ann_module4 = import_helper.import_fresh_module('test.ann_module4') + ann_module4 = import_helper.import_fresh_module( + 'test.typinganndata.ann_module4', + ) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index e880c3f1ac875e..72488b2bb6b4ff 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -1,7 +1,8 @@ # Python test set -- part 2, opcodes import unittest -from test import ann_module, support +from test import support +from test.typinganndata import ann_module class OpcodeTest(unittest.TestCase): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fa39c796197959..38baf9546f8b0f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5377,7 +5377,7 @@ def test_errors(self): # We need this to make sure that `@no_type_check` respects `__module__` attr: -from test import ann_module8 +from test.typinganndata import ann_module8 @no_type_check class NoTypeCheck_Outer: @@ -5968,7 +5968,9 @@ def test_overload_registry_repeated(self): # Definitions needed for features introduced in Python 3.6 -from test import ann_module, ann_module2, ann_module3, ann_module5, ann_module6 +from test.typinganndata import ( + ann_module, ann_module2, ann_module3, ann_module5, ann_module6, +) T_a = TypeVar('T_a') diff --git a/Lib/test/ann_module.py b/Lib/test/typinganndata/ann_module.py similarity index 100% rename from Lib/test/ann_module.py rename to Lib/test/typinganndata/ann_module.py diff --git a/Lib/test/ann_module2.py b/Lib/test/typinganndata/ann_module2.py similarity index 100% rename from Lib/test/ann_module2.py rename to Lib/test/typinganndata/ann_module2.py diff --git a/Lib/test/ann_module3.py b/Lib/test/typinganndata/ann_module3.py similarity index 100% rename from Lib/test/ann_module3.py rename to Lib/test/typinganndata/ann_module3.py diff --git a/Lib/test/ann_module4.py b/Lib/test/typinganndata/ann_module4.py similarity index 100% rename from Lib/test/ann_module4.py rename to Lib/test/typinganndata/ann_module4.py diff --git a/Lib/test/ann_module5.py b/Lib/test/typinganndata/ann_module5.py similarity index 100% rename from Lib/test/ann_module5.py rename to Lib/test/typinganndata/ann_module5.py diff --git a/Lib/test/ann_module6.py b/Lib/test/typinganndata/ann_module6.py similarity index 100% rename from Lib/test/ann_module6.py rename to Lib/test/typinganndata/ann_module6.py diff --git a/Lib/test/ann_module7.py b/Lib/test/typinganndata/ann_module7.py similarity index 100% rename from Lib/test/ann_module7.py rename to Lib/test/typinganndata/ann_module7.py diff --git a/Lib/test/ann_module8.py b/Lib/test/typinganndata/ann_module8.py similarity index 100% rename from Lib/test/ann_module8.py rename to Lib/test/typinganndata/ann_module8.py From 2135bcd3ca9538c6782129f9a5837d62c2036102 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Thu, 24 Aug 2023 00:45:20 +0900 Subject: [PATCH 275/598] gh-107265: Ensure de_instrument does not handle ENTER_EXECUTOR (#108366) --- Python/instrumentation.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a7a5b4a5dc5f6e..f77c2e696f4534 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -564,6 +564,7 @@ de_instrument(PyCodeObject *code, int i, int event) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; + assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; From 422f81b5d2359063826b8561f698d57e94f6a5d8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Aug 2023 20:12:42 +0300 Subject: [PATCH 276/598] gh-107298: Fix a few more refs in the C API docs (GH-108361) gh-107298: Fix few more refs in the C API docs --- Doc/c-api/typeobj.rst | 2 +- Doc/whatsnew/2.6.rst | 2 +- Doc/whatsnew/2.7.rst | 16 ++++++++-------- Doc/whatsnew/3.4.rst | 2 +- Doc/whatsnew/3.7.rst | 6 +++--- Misc/NEWS.d/3.11.0b1.rst | 4 ++-- Misc/NEWS.d/3.7.0a4.rst | 2 +- Misc/NEWS.d/3.8.0a4.rst | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index cd037b4de882e1..acaf0eced35b3f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1403,7 +1403,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) are allowed to be removed even if the instance is still alive). Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`local_traverse` to have these specific names; don't name them just + :c:func:`!local_traverse` to have these specific names; don't name them just anything. Instances of :ref:`heap-allocated types ` hold a reference to diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 8b3d3a324f68f6..beba4428e67c3f 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -121,7 +121,7 @@ about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available to Python code as the boolean variable :data:`sys.py3kwarning`, -and to C extension code as :c:data:`Py_Py3kWarningFlag`. +and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 37e2d04d516ab0..e82e8e4db1abcd 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2155,13 +2155,13 @@ Changes to Python's build process and to the C API include: :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. -* New function: :c:func:`PySys_SetArgvEx` sets the value of +* New function: :c:func:`!PySys_SetArgvEx` sets the value of ``sys.argv`` and can optionally update ``sys.path`` to include the directory containing the script named by ``sys.argv[0]`` depending on the value of an *updatepath* parameter. This function was added to close a security hole for applications - that embed Python. The old function, :c:func:`PySys_SetArgv`, would + that embed Python. The old function, :c:func:`!PySys_SetArgv`, would always update ``sys.path``, and sometimes it would add the current directory. This meant that, if you ran an application embedding Python in a directory controlled by someone else, attackers could @@ -2169,8 +2169,8 @@ Changes to Python's build process and to the C API include: :file:`os.py`) that your application would then import and run. If you maintain a C/C++ application that embeds Python, check - whether you're calling :c:func:`PySys_SetArgv` and carefully consider - whether the application should be using :c:func:`PySys_SetArgvEx` + whether you're calling :c:func:`!PySys_SetArgv` and carefully consider + whether the application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. Security issue reported as `CVE-2008-5983 @@ -2545,11 +2545,11 @@ For C extensions: For applications that embed Python: -* The :c:func:`PySys_SetArgvEx` function was added, letting +* The :c:func:`!PySys_SetArgvEx` function was added, letting applications close a security hole when the existing - :c:func:`PySys_SetArgv` function was used. Check whether you're - calling :c:func:`PySys_SetArgv` and carefully consider whether the - application should be using :c:func:`PySys_SetArgvEx` with + :c:func:`!PySys_SetArgv` function was used. Check whether you're + calling :c:func:`!PySys_SetArgv` and carefully consider whether the + application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. .. ====================================================================== diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 794271f3c32b89..a36e9fa852723a 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -1891,7 +1891,7 @@ Other Build and C API Changes allowing retrieval of function pointers from named type slots when using the limited API. (Contributed by Martin von Löwis in :issue:`17162`.) -* The new :c:func:`Py_SetStandardStreamEncoding` pre-initialization API +* The new :c:func:`!Py_SetStandardStreamEncoding` pre-initialization API allows applications embedding the CPython interpreter to reliably force a particular encoding and error handler for the standard streams. (Contributed by Bastien Montagne and Nick Coghlan in :issue:`16129`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 218a37cd264c7c..eb083737b0b7ed 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2495,12 +2495,12 @@ either in embedding applications, or in CPython itself. :issue:`22257`, and further updated by Nick, Eric, and Victor Stinner in a number of other issues). Some known details affected: -* :c:func:`PySys_AddWarnOptionUnicode` is not currently usable by embedding +* :c:func:`!PySys_AddWarnOptionUnicode` is not currently usable by embedding applications due to the requirement to create a Unicode object prior to - calling ``Py_Initialize``. Use :c:func:`PySys_AddWarnOption` instead. + calling ``Py_Initialize``. Use :c:func:`!PySys_AddWarnOption` instead. * warnings filters added by an embedding application with - :c:func:`PySys_AddWarnOption` should now more consistently take precedence + :c:func:`!PySys_AddWarnOption` should now more consistently take precedence over the default filters set by the interpreter Due to changes in the way the default warnings filters are configured, diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 766ada4f8e1176..2bcccc7dae3734 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -2028,8 +2028,8 @@ https://gitlab.com/warsaw/pynche .. nonce: 3mQ54t .. section: C API -Deprecate the C functions: :c:func:`PySys_SetArgv`, -:c:func:`PySys_SetArgvEx`, :c:func:`PySys_SetPath`. Patch by Victor Stinner. +Deprecate the C functions: :c:func:`!PySys_SetArgv`, +:c:func:`!PySys_SetArgvEx`, :c:func:`!PySys_SetPath`. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index f19d1a1823584b..ebae046a7a6ba0 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -842,5 +842,5 @@ Moved the pygetopt.h header into internal/, since it has no public APIs. .. nonce: LbyQt6 .. section: C API -:c:func:`Py_SetProgramName` and :c:func:`Py_SetPythonHome` now take the +:c:func:`!Py_SetProgramName` and :c:func:`!Py_SetPythonHome` now take the ``const wchar *`` arguments instead of ``wchar *``. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 2ce60f39539e8c..da03d93eae3965 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -1344,7 +1344,7 @@ Fix the argument handling in Tools/scripts/lll.py. .. nonce: vghb86 .. section: C API -Fix memory leak in :c:func:`Py_SetStandardStreamEncoding`: release memory if +Fix memory leak in :c:func:`!Py_SetStandardStreamEncoding`: release memory if the function is called twice. .. From 72119d16a5f658939809febef29dadeca02cf34d Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:39:00 +0100 Subject: [PATCH 277/598] gh-105481: remove regen-opcode. Generated _PyOpcode_Caches in regen-cases. (#108367) --- Include/internal/pycore_opcode.h | 40 ----------- Include/internal/pycore_opcode_metadata.h | 20 ++++++ Include/internal/pycore_opcode_utils.h | 3 +- Lib/opcode.py | 54 +++++++-------- Makefile.pre.in | 14 +--- ...-08-23-14-54-15.gh-issue-105481.40q-c4.rst | 2 + Objects/codeobject.c | 3 +- Objects/frameobject.c | 3 +- PCbuild/regen.targets | 14 +--- Python/assemble.c | 3 +- Python/ceval.c | 3 +- Python/compile.c | 1 + Python/executor.c | 2 + Python/instrumentation.c | 6 +- Python/optimizer.c | 1 - Python/optimizer_analysis.c | 1 - Python/specialize.c | 5 +- Tools/build/generate_opcode_h.py | 67 ------------------- Tools/cases_generator/generate_cases.py | 12 ++++ 19 files changed, 78 insertions(+), 176 deletions(-) delete mode 100644 Include/internal/pycore_opcode.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst delete mode 100644 Tools/build/generate_opcode_h.py diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h deleted file mode 100644 index b47e796485236c..00000000000000 --- a/Include/internal/pycore_opcode.h +++ /dev/null @@ -1,40 +0,0 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" - -extern const uint8_t _PyOpcode_Caches[256]; - -#ifdef NEED_OPCODE_TABLES - -const uint8_t _PyOpcode_Caches[256] = { - [LOAD_GLOBAL] = 4, - [BINARY_OP] = 1, - [UNPACK_SEQUENCE] = 1, - [COMPARE_OP] = 1, - [BINARY_SUBSCR] = 1, - [FOR_ITER] = 1, - [LOAD_SUPER_ATTR] = 1, - [LOAD_ATTR] = 9, - [STORE_ATTR] = 4, - [CALL] = 3, - [STORE_SUBSCR] = 1, - [SEND] = 1, - [JUMP_BACKWARD] = 1, - [TO_BOOL] = 3, -}; -#endif // NEED_OPCODE_TABLES - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e35db0c4c5a59d..cc8894ad53933f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1814,6 +1814,26 @@ const char *const _PyOpcode_OpName[268] = { }; #endif // NEED_OPCODE_METADATA +extern const uint8_t _PyOpcode_Caches[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Caches[256] = { + [TO_BOOL] = 3, + [BINARY_OP] = 1, + [BINARY_SUBSCR] = 1, + [STORE_SUBSCR] = 1, + [SEND] = 1, + [UNPACK_SEQUENCE] = 1, + [STORE_ATTR] = 4, + [LOAD_GLOBAL] = 4, + [LOAD_SUPER_ATTR] = 1, + [LOAD_ATTR] = 9, + [COMPARE_OP] = 1, + [FOR_ITER] = 1, + [CALL] = 3, + [JUMP_BACKWARD] = 1, +}; +#endif // NEED_OPCODE_METADATA + extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index f17612908cebdc..c4acb00a4b291e 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,8 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // JUMP_FORWARD - +#include "opcode_ids.h" #define MAX_REAL_OPCODE 254 diff --git a/Lib/opcode.py b/Lib/opcode.py index f8487522bfdc69..386a2fba396a6a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -5,48 +5,40 @@ """ -# Note that __all__ is further extended below -__all__ = ["cmp_op", "stack_effect", "hascompare"] +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] import _opcode from _opcode import stack_effect -import sys -# The build uses older versions of Python which do not have _opcode_metadata -if sys.version_info[:2] >= (3, 13): - from _opcode_metadata import _specializations, _specialized_opmap - from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE - EXTENDED_ARG = opmap['EXTENDED_ARG'] +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +EXTENDED_ARG = opmap['EXTENDED_ARG'] - opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] - for op, i in opmap.items(): - opname[i] = op - - __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"]) +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] +for op, i in opmap.items(): + opname[i] = op cmp_op = ('<', '<=', '==', '!=', '>', '>=') -# The build uses older versions of Python which do not have _opcode.has_* functions -if sys.version_info[:2] >= (3, 13): - # These lists are documented as part of the dis module's API - hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] - hasconst = [op for op in opmap.values() if _opcode.has_const(op)] - hasname = [op for op in opmap.values() if _opcode.has_name(op)] - hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] - hasjrel = hasjump # for backward compatibility - hasjabs = [] - hasfree = [op for op in opmap.values() if _opcode.has_free(op)] - haslocal = [op for op in opmap.values() if _opcode.has_local(op)] - hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] - __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", - "hasjabs", "hasfree", "haslocal", "hasexc"]) - _intrinsic_1_descs = _opcode.get_intrinsic1_descs() - _intrinsic_2_descs = _opcode.get_intrinsic2_descs() - _nb_ops = _opcode.get_nb_ops() +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_nb_ops = _opcode.get_nb_ops() - hascompare = [opmap["COMPARE_OP"]] +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { diff --git a/Makefile.pre.in b/Makefile.pre.in index d66764e6165409..54b64e123cd7bc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1321,7 +1321,7 @@ regen-limited-abi: all # Regenerate all generated files .PHONY: regen-all -regen-all: regen-cases regen-opcode regen-typeslots \ +regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -1425,15 +1425,6 @@ regen-ast: $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast_state.h $(srcdir)/Include/internal/pycore_ast_state.h.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new -.PHONY: regen-opcode -regen-opcode: - # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py - # using Tools/build/generate_opcode_h.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ - $(srcdir)/Lib/opcode.py \ - $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new - .PHONY: regen-token regen-token: # Regenerate Doc/library/token-list.inc from Grammar/Tokens @@ -1651,6 +1642,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -1790,7 +1782,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object_state.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst new file mode 100644 index 00000000000000..19746ebb701d02 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst @@ -0,0 +1,2 @@ +The regen-opcode build stage was removed and its work is now done in +regen-cases. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index dca5804a91d2cd..70a0c2ebd66b24 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -6,8 +6,7 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs -#include "pycore_opcode.h" // _PyOpcode_Caches -#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 80e118e8a8aa93..28f5a5a1222806 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -6,8 +6,7 @@ #include "pycore_function.h" // _PyFunction_FromConstructor() #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_opcode.h" // _PyOpcode_Caches -#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "frameobject.h" // PyFrameObject diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 2ff18a8966f557..cc9469c7ddd726 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -13,8 +13,6 @@ <_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c"> -C - <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -34,7 +32,7 @@ - @@ -55,14 +53,6 @@ WorkingDirectory="$(PySourcePath)" /> - - - - - @@ -89,7 +79,7 @@ + DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> diff --git a/Python/assemble.c b/Python/assemble.c index 4f66cf294e38c9..c770fd108d41d6 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -3,9 +3,8 @@ #include "Python.h" #include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" -#include "pycore_opcode.h" // _PyOpcode_Caches[] and opcode category macros #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE -#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR +#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches #define DEFAULT_CODE_SIZE 128 diff --git a/Python/ceval.c b/Python/ceval.c index f7dfaebcdd8f84..55dfe6b1efc475 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -14,8 +14,7 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_opcode_metadata.h" +#include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/compile.c b/Python/compile.c index b67a1885ddc983..6b816b4c6eda6c 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -24,6 +24,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ast.h" // _PyAST_GetDocString() #define NEED_OPCODE_TABLES #include "pycore_opcode_utils.h" diff --git a/Python/executor.c b/Python/executor.c index 88c039da8539e2..0ff5106fe446ff 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -1,5 +1,7 @@ #include "Python.h" +#include "opcode.h" + #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_dict.h" diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f77c2e696f4534..8c7a3a06c9b938 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1,4 +1,7 @@ #include "Python.h" + +#include "opcode_ids.h" + #include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" @@ -6,8 +9,7 @@ #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" #include "pycore_object.h" -#include "pycore_opcode.h" -#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/optimizer.c b/Python/optimizer.c index 57518404c3f19d..bbc125954c701e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,7 +1,6 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e48e018052c712..2d177f14ff268b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,7 +1,6 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/specialize.c b/Python/specialize.c index 2d514c0dc476d3..a467f163f2ca9d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1,4 +1,7 @@ #include "Python.h" + +#include "opcode.h" + #include "pycore_code.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" @@ -7,7 +10,7 @@ #include "pycore_long.h" #include "pycore_moduleobject.h" #include "pycore_object.h" -#include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py deleted file mode 100644 index 643918c1bc2ce4..00000000000000 --- a/Tools/build/generate_opcode_h.py +++ /dev/null @@ -1,67 +0,0 @@ -# This script generates the pycore_opcode.h header file. - -import sys -import tokenize - -SCRIPT_NAME = "Tools/build/generate_opcode_h.py" -PYTHON_OPCODE = "Lib/opcode.py" - -internal_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" {{ -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" -""".lstrip() - -internal_footer = """ -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H -""" - -DEFINE = "#define {:<38} {:>3}\n" - -UINT32_MASK = (1<<32)-1 - -def get_python_module_dict(filename): - mod = {} - with tokenize.open(filename) as fp: - code = fp.read() - exec(code, mod) - return mod - -def main(opcode_py, - internal_opcode_h='Include/internal/pycore_opcode.h'): - - opcode = get_python_module_dict(opcode_py) - - with open(internal_opcode_h, 'w') as iobj: - iobj.write(internal_header) - - iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") - iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") - - iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") - for name, entries in opcode["_inline_cache_entries"].items(): - iobj.write(f" [{name}] = {entries},\n") - iobj.write("};\n") - - iobj.write("#endif // NEED_OPCODE_TABLES\n") - - iobj.write(internal_footer) - - print(f"{internal_opcode_h} regenerated from {opcode_py}") - - -if __name__ == '__main__': - main(sys.argv[1], sys.argv[2]) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f605dcc5e4632f..8f9a6502e529c7 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -540,6 +540,18 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No for name in self.opmap: self.out.emit(f'[{name}] = "{name}",') + with self.metadata_item( + f"const uint8_t _PyOpcode_Caches[256]", + "=", + ";", + ): + for name, _ in self.families.items(): + instr = self.instrs[name] + if instr.cache_offset > 0: + self.out.emit(f'[{name}] = {instr.cache_offset},') + # Irregular case: + self.out.emit('[JUMP_BACKWARD] = 1,') + deoptcodes = {} for name, op in self.opmap.items(): if op < 256: From 5d1871576500adc4ebaa7f59b8559605a57ad36b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 23 Aug 2023 20:00:07 +0200 Subject: [PATCH 278/598] gh-107811: tarfile: treat overflow in UID/GID as failure to set it (#108369) --- Lib/tarfile.py | 3 ++- .../Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index a835d00c90c92c..726f9f50ba2e72 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2557,7 +2557,8 @@ def chown(self, tarinfo, targetpath, numeric_owner): os.lchown(targetpath, u, g) else: os.chown(targetpath, u, g) - except OSError as e: + except (OSError, OverflowError) as e: + # OverflowError can be raised if an ID doesn't fit in `id_t` raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): diff --git a/Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst b/Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst new file mode 100644 index 00000000000000..ffca4131db228b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst @@ -0,0 +1,3 @@ +:mod:`tarfile`: extraction of members with overly large UID or GID (e.g. on +an OS with 32-bit :c:type:`!id_t`) now fails in the same way as failing to +set the ID. From 1700d34d314f5304a7a75363bda295a8c15c371f Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 23 Aug 2023 16:27:35 -0400 Subject: [PATCH 279/598] gh-77377: Ensure multiprocessing SemLock is valid for spawn-based Process before serializing it (#107275) Ensure multiprocessing SemLock is valid for spawn Process before serializing it. Creating a multiprocessing SemLock with a fork context, and then trying to pass it to a spawn-created Process, would segfault if not detected early. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Antoine Pitrou --- Lib/multiprocessing/synchronize.py | 9 ++++++-- Lib/test/_test_multiprocessing.py | 22 +++++++++++++++++++ ...3-07-25-22-35-35.gh-issue-77377.EHAbXx.rst | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 42624b543601a1..2328d332123082 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -50,8 +50,8 @@ class SemLock(object): def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() - name = ctx.get_start_method() - unlink_now = sys.platform == 'win32' or name == 'fork' + self.is_fork_ctx = ctx.get_start_method() == 'fork' + unlink_now = sys.platform == 'win32' or self.is_fork_ctx for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( @@ -103,6 +103,11 @@ def __getstate__(self): if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: + if self.is_fork_ctx: + raise RuntimeError('A SemLock created in a fork context is being ' + 'shared with a process in a spawn context. This is ' + 'not supported. Please use the same context to create ' + 'multiprocessing objects and Process.') h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 10754964e73bc5..a826e31f5ee079 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5421,6 +5421,28 @@ def test_preload_resources(self): print(err) self.fail("failed spawning forkserver or grandchild") + @unittest.skipIf(sys.platform == "win32", + "Only Spawn on windows so no risk of mixing") + @only_run_in_spawn_testsuite("avoids redundant testing.") + def test_mixed_startmethod(self): + # Fork-based locks cannot be used with spawned process + for process_method in ["spawn", "forkserver"]: + queue = multiprocessing.get_context("fork").Queue() + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + err_msg = "A SemLock created in a fork" + with self.assertRaisesRegex(RuntimeError, err_msg): + p.start() + + # non-fork-based locks can be used with all other start methods + for queue_method in ["spawn", "forkserver"]: + for process_method in multiprocessing.get_all_start_methods(): + queue = multiprocessing.get_context(queue_method).Queue() + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + p.start() + p.join() + @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst new file mode 100644 index 00000000000000..194851dea13352 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst @@ -0,0 +1 @@ +Ensure that multiprocessing synchronization objects created in a fork context are not sent to a different process created in a spawn context. This changes a segfault into an actionable RuntimeError in the parent process. From 4dc9f4893084f7c3acf78a0384620cd44f604a0d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 22:59:00 +0200 Subject: [PATCH 280/598] gh-108308: Replace _PyDict_GetItemStringWithError() (#108372) Replace _PyDict_GetItemStringWithError() calls with PyDict_GetItemStringRef() which returns a strong reference to the item. Co-authored-by: Serhiy Storchaka --- Objects/structseq.c | 25 ++++++++++--------------- Python/codecs.c | 12 +++++------- Python/import.c | 8 +++----- Python/pylifecycle.c | 26 ++++++++++++++------------ 4 files changed, 32 insertions(+), 39 deletions(-) diff --git a/Objects/structseq.c b/Objects/structseq.c index 95c4c15710d169..6c07e6366293f9 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -8,7 +8,6 @@ */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -149,7 +148,6 @@ static PyObject * structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) /*[clinic end generated code: output=baa082e788b171da input=90532511101aa3fb]*/ { - PyObject *ob; PyStructSequence *res = NULL; Py_ssize_t len, min_len, max_len, i, n_unnamed_fields; @@ -219,21 +217,18 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) } Py_DECREF(arg); for (; i < max_len; ++i) { - if (dict == NULL) { - ob = Py_None; - } - else { - ob = _PyDict_GetItemStringWithError(dict, - type->tp_members[i-n_unnamed_fields].name); - if (ob == NULL) { - if (PyErr_Occurred()) { - Py_DECREF(res); - return NULL; - } - ob = Py_None; + PyObject *ob = NULL; + if (dict != NULL) { + const char *name = type->tp_members[i-n_unnamed_fields].name; + if (PyDict_GetItemStringRef(dict, name, &ob) < 0) { + Py_DECREF(res); + return NULL; } } - res->ob_item[i] = Py_NewRef(ob); + if (ob == NULL) { + ob = Py_NewRef(Py_None); + } + res->ob_item[i] = ob; } _PyObject_GC_TRACK(res); diff --git a/Python/codecs.c b/Python/codecs.c index 3c418512e3aa0b..87ae896b8e7718 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -10,7 +10,6 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -618,20 +617,19 @@ int PyCodec_RegisterError(const char *name, PyObject *error) the error handling callback for strict encoding will be returned. */ PyObject *PyCodec_LookupError(const char *name) { - PyObject *handler = NULL; - PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) return NULL; if (name==NULL) name = "strict"; - handler = _PyDict_GetItemStringWithError(interp->codec_error_registry, name); - if (handler) { - Py_INCREF(handler); + PyObject *handler; + if (PyDict_GetItemStringRef(interp->codec_error_registry, name, &handler) < 0) { + return NULL; } - else if (!PyErr_Occurred()) { + if (handler == NULL) { PyErr_Format(PyExc_LookupError, "unknown error handler name '%.400s'", name); + return NULL; } return handler; } diff --git a/Python/import.c b/Python/import.c index 4abb2f6a06b48c..5681deb3c4fc73 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,7 +1,6 @@ /* Module definition and import implementation */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -2435,12 +2434,11 @@ int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp) { // Get the __import__ function - PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, - "__import__"); - if (import_func == NULL) { + PyObject *import_func; + if (PyDict_GetItemStringRef(interp->builtins, "__import__", &import_func) <= 0) { return -1; } - IMPORT_FUNC(interp) = Py_NewRef(import_func); + IMPORT_FUNC(interp) = import_func; return 0; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8c321fbcfe4104..18614264989185 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1372,17 +1372,17 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) if (verbose) { PySys_WriteStderr("# restore sys.%s\n", name); } - PyObject *value = _PyDict_GetItemStringWithError(interp->sysdict, - orig_name); + PyObject *value; + if (PyDict_GetItemStringRef(interp->sysdict, orig_name, &value) < 0) { + PyErr_WriteUnraisable(NULL); + } if (value == NULL) { - if (_PyErr_Occurred(tstate)) { - PyErr_WriteUnraisable(NULL); - } - value = Py_None; + value = Py_NewRef(Py_None); } if (PyDict_SetItemString(interp->sysdict, name, value) < 0) { PyErr_WriteUnraisable(NULL); } + Py_DECREF(value); } } @@ -2207,7 +2207,7 @@ _Py_IsInterpreterFinalizing(PyInterpreterState *interp) static PyStatus add_main_module(PyInterpreterState *interp) { - PyObject *m, *d, *loader, *ann_dict; + PyObject *m, *d, *ann_dict; m = PyImport_AddModule("__main__"); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); @@ -2239,11 +2239,13 @@ add_main_module(PyInterpreterState *interp) * will be set if __main__ gets further initialized later in the startup * process. */ - loader = _PyDict_GetItemStringWithError(d, "__loader__"); - if (loader == NULL || loader == Py_None) { - if (PyErr_Occurred()) { - return _PyStatus_ERR("Failed to test __main__.__loader__"); - } + PyObject *loader; + if (PyDict_GetItemStringRef(d, "__loader__", &loader) < 0) { + return _PyStatus_ERR("Failed to test __main__.__loader__"); + } + int has_loader = !(loader == NULL || loader == Py_None); + Py_XDECREF(loader); + if (!has_loader) { PyObject *loader = _PyImport_GetImportlibLoader(interp, "BuiltinImporter"); if (loader == NULL) { From ec3527d19627e62212f792513e6cab62b6f0f46a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 23:12:08 +0200 Subject: [PATCH 281/598] gh-108308: config_dict_get() uses PyDict_GetItemRef() (#108371) Replace _PyDict_GetItemStringWithError() with PyDict_GetItemRef() in config_dict_get() to get a strong reference to the item. --- Python/initconfig.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index c017abeb90563e..39d21adf546466 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_fileutils.h" // _Py_HasFileSystemDefaultEncodeErrors #include "pycore_getopt.h" // _PyOS_GetOpt() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -1065,8 +1064,11 @@ _PyConfig_AsDict(const PyConfig *config) static PyObject* config_dict_get(PyObject *dict, const char *name) { - PyObject *item = _PyDict_GetItemStringWithError(dict, name); - if (item == NULL && !PyErr_Occurred()) { + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return NULL; + } + if (item == NULL) { PyErr_Format(PyExc_ValueError, "missing config key: %s", name); return NULL; } @@ -1096,6 +1098,7 @@ config_dict_get_int(PyObject *dict, const char *name, int *result) return -1; } int value = _PyLong_AsInt(item); + Py_DECREF(item); if (value == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { config_dict_invalid_type(name); @@ -1118,6 +1121,7 @@ config_dict_get_ulong(PyObject *dict, const char *name, unsigned long *result) return -1; } unsigned long value = PyLong_AsUnsignedLong(item); + Py_DECREF(item); if (value == (unsigned long)-1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { config_dict_invalid_type(name); @@ -1140,27 +1144,33 @@ config_dict_get_wstr(PyObject *dict, const char *name, PyConfig *config, if (item == NULL) { return -1; } + PyStatus status; if (item == Py_None) { status = PyConfig_SetString(config, result, NULL); } else if (!PyUnicode_Check(item)) { config_dict_invalid_type(name); - return -1; + goto error; } else { wchar_t *wstr = PyUnicode_AsWideCharString(item, NULL); if (wstr == NULL) { - return -1; + goto error; } status = PyConfig_SetString(config, result, wstr); PyMem_Free(wstr); } if (_PyStatus_EXCEPTION(status)) { PyErr_NoMemory(); - return -1; + goto error; } + Py_DECREF(item); return 0; + +error: + Py_DECREF(item); + return -1; } @@ -1174,6 +1184,7 @@ config_dict_get_wstrlist(PyObject *dict, const char *name, PyConfig *config, } if (!PyList_CheckExact(list)) { + Py_DECREF(list); config_dict_invalid_type(name); return -1; } @@ -1207,10 +1218,12 @@ config_dict_get_wstrlist(PyObject *dict, const char *name, PyConfig *config, goto error; } _PyWideStringList_Clear(&wstrlist); + Py_DECREF(list); return 0; error: _PyWideStringList_Clear(&wstrlist); + Py_DECREF(list); return -1; } From 592bacb6fc0833336c0453e818e9b95016e9fd47 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 23:57:11 +0200 Subject: [PATCH 282/598] gh-108342: Make ssl TestPreHandshakeClose more reliable (#108370) * In preauth tests of test_ssl, explicitly break reference cycles invoving SingleConnectionTestServerThread to make sure that the thread is deleted. Otherwise, the test marks the environment as altered because the threading module sees a "dangling thread" (SingleConnectionTestServerThread). This test leak was introduced by the test added for the fix of issue gh-108310. * Use support.SHORT_TIMEOUT instead of hardcoded 1.0 or 2.0 seconds timeout. * SingleConnectionTestServerThread.run() catchs TimeoutError * Fix a race condition (missing synchronization) in test_preauth_data_to_tls_client(): the server now waits until the client connect() completed in call_after_accept(). * test_https_client_non_tls_response_ignored() calls server.join() explicitly. * Replace "localhost" with server.listener.getsockname()[0]. --- Lib/test/test_ssl.py | 102 ++++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index ad5377ec059c0b..4e49dc5640d3f5 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4672,12 +4672,16 @@ class TestPreHandshakeClose(unittest.TestCase): class SingleConnectionTestServerThread(threading.Thread): - def __init__(self, *, name, call_after_accept): + def __init__(self, *, name, call_after_accept, timeout=None): self.call_after_accept = call_after_accept self.received_data = b'' # set by .run() self.wrap_error = None # set by .run() self.listener = None # set by .start() self.port = None # set by .start() + if timeout is None: + self.timeout = support.SHORT_TIMEOUT + else: + self.timeout = timeout super().__init__(name=name) def __enter__(self): @@ -4700,13 +4704,19 @@ def start(self): self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) self.listener = socket.socket() self.port = socket_helper.bind_port(self.listener) - self.listener.settimeout(2.0) + self.listener.settimeout(self.timeout) self.listener.listen(1) super().start() def run(self): - conn, address = self.listener.accept() - self.listener.close() + try: + conn, address = self.listener.accept() + except TimeoutError: + # on timeout, just close the listener + return + finally: + self.listener.close() + with conn: if self.call_after_accept(conn): return @@ -4734,8 +4744,13 @@ def non_linux_skip_if_other_okay_error(self, err): # we're specifically trying to test. The way this test is written # is known to work on Linux. We'll skip it anywhere else that it # does not present as doing so. - self.skipTest(f"Could not recreate conditions on {sys.platform}:" - f" {err=}") + try: + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + finally: + # gh-108342: Explicitly break the reference cycle + err = None + # If maintaining this conditional winds up being a problem. # just turn this into an unconditional skip anything but Linux. # The important thing is that our CI has the logic covered. @@ -4746,7 +4761,7 @@ def test_preauth_data_to_tls_server(self): def call_after_accept(unused): server_accept_called.set() - if not ready_for_server_wrap_socket.wait(2.0): + if not ready_for_server_wrap_socket.wait(support.SHORT_TIMEOUT): raise RuntimeError("wrap_socket event never set, test may fail.") return False # Tell the server thread to continue. @@ -4767,20 +4782,31 @@ def call_after_accept(unused): ready_for_server_wrap_socket.set() server.join() + wrap_error = server.wrap_error - self.assertEqual(b"", server.received_data) - self.assertIsInstance(wrap_error, OSError) # All platforms. - self.non_linux_skip_if_other_okay_error(wrap_error) - self.assertIsInstance(wrap_error, ssl.SSLError) - self.assertIn("before TLS handshake with data", wrap_error.args[1]) - self.assertIn("before TLS handshake with data", wrap_error.reason) - self.assertNotEqual(0, wrap_error.args[0]) - self.assertIsNone(wrap_error.library, msg="attr must exist") + server.wrap_error = None + try: + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + finally: + # gh-108342: Explicitly break the reference cycle + wrap_error = None + server = None def test_preauth_data_to_tls_client(self): + server_can_continue_with_wrap_socket = threading.Event() client_can_continue_with_wrap_socket = threading.Event() def call_after_accept(conn_to_client): + if not server_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT): + print("ERROR: test client took too long") + # This forces an immediate connection close via RST on .close(). set_socket_so_linger_on_with_zero_timeout(conn_to_client) conn_to_client.send( @@ -4800,8 +4826,10 @@ def call_after_accept(conn_to_client): with socket.socket() as client: client.connect(server.listener.getsockname()) - if not client_can_continue_with_wrap_socket.wait(2.0): - self.fail("test server took too long.") + server_can_continue_with_wrap_socket.set() + + if not client_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT): + self.fail("test server took too long") ssl_ctx = ssl.create_default_context() try: tls_client = ssl_ctx.wrap_socket( @@ -4815,24 +4843,31 @@ def call_after_accept(conn_to_client): tls_client.close() server.join() - self.assertEqual(b"", received_data) - self.assertIsInstance(wrap_error, OSError) # All platforms. - self.non_linux_skip_if_other_okay_error(wrap_error) - self.assertIsInstance(wrap_error, ssl.SSLError) - self.assertIn("before TLS handshake with data", wrap_error.args[1]) - self.assertIn("before TLS handshake with data", wrap_error.reason) - self.assertNotEqual(0, wrap_error.args[0]) - self.assertIsNone(wrap_error.library, msg="attr must exist") + try: + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + finally: + # gh-108342: Explicitly break the reference cycle + wrap_error = None + server = None def test_https_client_non_tls_response_ignored(self): - server_responding = threading.Event() class SynchronizedHTTPSConnection(http.client.HTTPSConnection): def connect(self): + # Call clear text HTTP connect(), not the encrypted HTTPS (TLS) + # connect(): wrap_socket() is called manually below. http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. - if not server_responding.wait(1.0) and support.verbose: + if not server_responding.wait(support.SHORT_TIMEOUT) and support.verbose: sys.stdout.write("server_responding event never set.") self.sock = self._context.wrap_socket( self.sock, server_hostname=self.host) @@ -4847,28 +4882,33 @@ def call_after_accept(conn_to_client): server_responding.set() return True # Tell the server to stop. + timeout = 2.0 server = self.SingleConnectionTestServerThread( call_after_accept=call_after_accept, - name="non_tls_http_RST_responder") + name="non_tls_http_RST_responder", + timeout=timeout) self.enterContext(server) # starts it & unittest.TestCase stops it. # Redundant; call_after_accept sets SO_LINGER on the accepted conn. set_socket_so_linger_on_with_zero_timeout(server.listener) connection = SynchronizedHTTPSConnection( - f"localhost", + server.listener.getsockname()[0], port=server.port, context=ssl.create_default_context(), - timeout=2.0, + timeout=timeout, ) + # There are lots of reasons this raises as desired, long before this # test was added. Sending the request requires a successful TLS wrapped # socket; that fails if the connection is broken. It may seem pointless # to test this. It serves as an illustration of something that we never # want to happen... properly not happening. - with self.assertRaises(OSError) as err_ctx: + with self.assertRaises(OSError): connection.request("HEAD", "/test", headers={"Host": "localhost"}) response = connection.getresponse() + server.join() + class TestEnumerations(unittest.TestCase): From 4890f65ecf119a18df926d42982c8c6d02f0b9fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Aug 2023 23:57:40 +0200 Subject: [PATCH 283/598] gh-108308: Use PyDict_GetItemRef() in moduleobject.c (#108381) Replace PyDict_GetItemWithError() with PyDict_GetItemRef() which returns a strong reference. Cleanup also the function: move variable definition to their first assignation, rename variable names to use name longer than 1 character. --- Objects/moduleobject.c | 68 +++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 7e890d021cb946..5b9f74dbddf031 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -510,25 +510,31 @@ PyModule_GetDict(PyObject *m) } PyObject* -PyModule_GetNameObject(PyObject *m) +PyModule_GetNameObject(PyObject *mod) { - PyObject *d; - PyObject *name; - if (!PyModule_Check(m)) { + if (!PyModule_Check(mod)) { PyErr_BadArgument(); return NULL; } - d = ((PyModuleObject *)m)->md_dict; - if (d == NULL || !PyDict_Check(d) || - (name = PyDict_GetItemWithError(d, &_Py_ID(__name__))) == NULL || - !PyUnicode_Check(name)) - { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_SystemError, "nameless module"); - } - return NULL; + PyObject *dict = ((PyModuleObject *)mod)->md_dict; // borrowed reference + if (dict == NULL || !PyDict_Check(dict)) { + goto error; + } + PyObject *name; + if (PyDict_GetItemRef(dict, &_Py_ID(__name__), &name) <= 0) { + goto error; + } + if (!PyUnicode_Check(name)) { + Py_DECREF(name); + goto error; + } + return name; + +error: + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "nameless module"); } - return Py_NewRef(name); + return NULL; } const char * @@ -544,25 +550,31 @@ PyModule_GetName(PyObject *m) } PyObject* -PyModule_GetFilenameObject(PyObject *m) +PyModule_GetFilenameObject(PyObject *mod) { - PyObject *d; - PyObject *fileobj; - if (!PyModule_Check(m)) { + if (!PyModule_Check(mod)) { PyErr_BadArgument(); return NULL; } - d = ((PyModuleObject *)m)->md_dict; - if (d == NULL || - (fileobj = PyDict_GetItemWithError(d, &_Py_ID(__file__))) == NULL || - !PyUnicode_Check(fileobj)) - { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_SystemError, "module filename missing"); - } - return NULL; + PyObject *dict = ((PyModuleObject *)mod)->md_dict; // borrowed reference + if (dict == NULL) { + goto error; } - return Py_NewRef(fileobj); + PyObject *fileobj; + if (PyDict_GetItemRef(dict, &_Py_ID(__file__), &fileobj) <= 0) { + goto error; + } + if (!PyUnicode_Check(fileobj)) { + Py_DECREF(fileobj); + goto error; + } + return fileobj; + +error: + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "module filename missing"); + } + return NULL; } const char * From b6be18812c68fce5ab56c266dc5fc5a3cceb09c0 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 23 Aug 2023 15:00:09 -0700 Subject: [PATCH 284/598] gh-70766: Mention the object getstate caveat in 3.11 What's new. (#108379) --- Doc/whatsnew/3.11.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 58ce881e9787f6..cc5cfee08d2b32 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -459,6 +459,10 @@ Other Language Changes :class:`collections.OrderedDict`, :class:`collections.deque`, :class:`weakref.WeakSet`, and :class:`datetime.tzinfo` now copies and pickles instance attributes implemented as :term:`slots <__slots__>`. + This change has an unintended side effect: It trips up a small minority + of existing Python projects not expecting :meth:`object.__getstate__` to + exist. See the later comments on :gh:`70766` for discussions of what + workarounds such code may need. (Contributed by Serhiy Storchaka in :issue:`26579`.) .. _whatsnew311-pythonsafepath: From 513c89d0122ff6245cfefbb49ef32bde8a2336f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 00:19:11 +0200 Subject: [PATCH 285/598] gh-108240: Add _PyCapsule_SetTraverse() internal function (#108339) The _socket extension uses _PyCapsule_SetTraverse() to visit and clear the socket type in the garbage collector. So the _socket.socket type can be cleared in some corner cases when it wasn't possible before. --- Include/pycapsule.h | 4 + Modules/socketmodule.c | 37 +++++++-- Objects/capsule.c | 166 +++++++++++++++++++++++++---------------- 3 files changed, 135 insertions(+), 72 deletions(-) diff --git a/Include/pycapsule.h b/Include/pycapsule.h index 929a9a685259fb..8660e83088edd6 100644 --- a/Include/pycapsule.h +++ b/Include/pycapsule.h @@ -48,6 +48,10 @@ PyAPI_FUNC(int) PyCapsule_SetName(PyObject *capsule, const char *name); PyAPI_FUNC(int) PyCapsule_SetContext(PyObject *capsule, void *context); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(int) _PyCapsule_SetTraverse(PyObject *op, traverseproc traverse_func, inquiry clear_func); +#endif + PyAPI_FUNC(void *) PyCapsule_Import( const char *name, /* UTF-8 encoded string */ int no_block); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bb5edc368decb3..392a56d5ab7d3e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7314,20 +7314,39 @@ os_init(void) } #endif +static int +sock_capi_traverse(PyObject *capsule, visitproc visit, void *arg) +{ + PySocketModule_APIObject *capi = PyCapsule_GetPointer(capsule, PySocket_CAPSULE_NAME); + assert(capi != NULL); + Py_VISIT(capi->Sock_Type); + return 0; +} + +static int +sock_capi_clear(PyObject *capsule) +{ + PySocketModule_APIObject *capi = PyCapsule_GetPointer(capsule, PySocket_CAPSULE_NAME); + assert(capi != NULL); + Py_CLEAR(capi->Sock_Type); + return 0; +} + static void -sock_free_api(PySocketModule_APIObject *capi) +sock_capi_free(PySocketModule_APIObject *capi) { - Py_DECREF(capi->Sock_Type); + Py_XDECREF(capi->Sock_Type); // sock_capi_free() can clear it Py_DECREF(capi->error); Py_DECREF(capi->timeout_error); PyMem_Free(capi); } static void -sock_destroy_api(PyObject *capsule) +sock_capi_destroy(PyObject *capsule) { void *capi = PyCapsule_GetPointer(capsule, PySocket_CAPSULE_NAME); - sock_free_api(capi); + assert(capi != NULL); + sock_capi_free(capi); } static PySocketModule_APIObject * @@ -7432,11 +7451,17 @@ socket_exec(PyObject *m) } PyObject *capsule = PyCapsule_New(capi, PySocket_CAPSULE_NAME, - sock_destroy_api); + sock_capi_destroy); if (capsule == NULL) { - sock_free_api(capi); + sock_capi_free(capi); goto error; } + if (_PyCapsule_SetTraverse(capsule, + sock_capi_traverse, sock_capi_clear) < 0) { + sock_capi_free(capi); + goto error; + } + if (PyModule_Add(m, PySocket_CAPI_NAME, capsule) < 0) { goto error; } diff --git a/Objects/capsule.c b/Objects/capsule.c index baaddb3f1f0849..aa320013c53460 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -9,18 +9,28 @@ typedef struct { const char *name; void *context; PyCapsule_Destructor destructor; + traverseproc traverse_func; + inquiry clear_func; } PyCapsule; static int -_is_legal_capsule(PyCapsule *capsule, const char *invalid_capsule) +_is_legal_capsule(PyObject *op, const char *invalid_capsule) { - if (!capsule || !PyCapsule_CheckExact(capsule) || capsule->pointer == NULL) { - PyErr_SetString(PyExc_ValueError, invalid_capsule); - return 0; + if (!op || !PyCapsule_CheckExact(op)) { + goto error; + } + PyCapsule *capsule = (PyCapsule *)op; + + if (capsule->pointer == NULL) { + goto error; } return 1; + +error: + PyErr_SetString(PyExc_ValueError, invalid_capsule); + return 0; } #define is_legal_capsule(capsule, name) \ @@ -50,7 +60,7 @@ PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) return NULL; } - capsule = PyObject_New(PyCapsule, &PyCapsule_Type); + capsule = PyObject_GC_New(PyCapsule, &PyCapsule_Type); if (capsule == NULL) { return NULL; } @@ -59,15 +69,18 @@ PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) capsule->name = name; capsule->context = NULL; capsule->destructor = destructor; + capsule->traverse_func = NULL; + capsule->clear_func = NULL; + // Only track the capsule if _PyCapsule_SetTraverse() is called return (PyObject *)capsule; } int -PyCapsule_IsValid(PyObject *o, const char *name) +PyCapsule_IsValid(PyObject *op, const char *name) { - PyCapsule *capsule = (PyCapsule *)o; + PyCapsule *capsule = (PyCapsule *)op; return (capsule != NULL && PyCapsule_CheckExact(capsule) && @@ -77,13 +90,12 @@ PyCapsule_IsValid(PyObject *o, const char *name) void * -PyCapsule_GetPointer(PyObject *o, const char *name) +PyCapsule_GetPointer(PyObject *op, const char *name) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_GetPointer")) { + if (!is_legal_capsule(op, "PyCapsule_GetPointer")) { return NULL; } + PyCapsule *capsule = (PyCapsule *)op; if (!name_matches(name, capsule->name)) { PyErr_SetString(PyExc_ValueError, "PyCapsule_GetPointer called with incorrect name"); @@ -95,52 +107,48 @@ PyCapsule_GetPointer(PyObject *o, const char *name) const char * -PyCapsule_GetName(PyObject *o) +PyCapsule_GetName(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_GetName")) { + if (!is_legal_capsule(op, "PyCapsule_GetName")) { return NULL; } + PyCapsule *capsule = (PyCapsule *)op; return capsule->name; } PyCapsule_Destructor -PyCapsule_GetDestructor(PyObject *o) +PyCapsule_GetDestructor(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_GetDestructor")) { + if (!is_legal_capsule(op, "PyCapsule_GetDestructor")) { return NULL; } + PyCapsule *capsule = (PyCapsule *)op; return capsule->destructor; } void * -PyCapsule_GetContext(PyObject *o) +PyCapsule_GetContext(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_GetContext")) { + if (!is_legal_capsule(op, "PyCapsule_GetContext")) { return NULL; } + PyCapsule *capsule = (PyCapsule *)op; return capsule->context; } int -PyCapsule_SetPointer(PyObject *o, void *pointer) +PyCapsule_SetPointer(PyObject *op, void *pointer) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!pointer) { - PyErr_SetString(PyExc_ValueError, "PyCapsule_SetPointer called with null pointer"); + if (!is_legal_capsule(op, "PyCapsule_SetPointer")) { return -1; } + PyCapsule *capsule = (PyCapsule *)op; - if (!is_legal_capsule(capsule, "PyCapsule_SetPointer")) { + if (!pointer) { + PyErr_SetString(PyExc_ValueError, "PyCapsule_SetPointer called with null pointer"); return -1; } @@ -150,13 +158,12 @@ PyCapsule_SetPointer(PyObject *o, void *pointer) int -PyCapsule_SetName(PyObject *o, const char *name) +PyCapsule_SetName(PyObject *op, const char *name) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_SetName")) { + if (!is_legal_capsule(op, "PyCapsule_SetName")) { return -1; } + PyCapsule *capsule = (PyCapsule *)op; capsule->name = name; return 0; @@ -164,13 +171,12 @@ PyCapsule_SetName(PyObject *o, const char *name) int -PyCapsule_SetDestructor(PyObject *o, PyCapsule_Destructor destructor) +PyCapsule_SetDestructor(PyObject *op, PyCapsule_Destructor destructor) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_SetDestructor")) { + if (!is_legal_capsule(op, "PyCapsule_SetDestructor")) { return -1; } + PyCapsule *capsule = (PyCapsule *)op; capsule->destructor = destructor; return 0; @@ -178,19 +184,36 @@ PyCapsule_SetDestructor(PyObject *o, PyCapsule_Destructor destructor) int -PyCapsule_SetContext(PyObject *o, void *context) +PyCapsule_SetContext(PyObject *op, void *context) { - PyCapsule *capsule = (PyCapsule *)o; - - if (!is_legal_capsule(capsule, "PyCapsule_SetContext")) { + if (!is_legal_capsule(op, "PyCapsule_SetContext")) { return -1; } + PyCapsule *capsule = (PyCapsule *)op; capsule->context = context; return 0; } +int +_PyCapsule_SetTraverse(PyObject *op, traverseproc traverse_func, inquiry clear_func) +{ + if (!is_legal_capsule(op, "_PyCapsule_SetTraverse")) { + return -1; + } + PyCapsule *capsule = (PyCapsule *)op; + + if (!PyObject_GC_IsTracked(op)) { + PyObject_GC_Track(op); + } + + capsule->traverse_func = traverse_func; + capsule->clear_func = clear_func; + return 0; +} + + void * PyCapsule_Import(const char *name, int no_block) { @@ -249,13 +272,14 @@ PyCapsule_Import(const char *name, int no_block) static void -capsule_dealloc(PyObject *o) +capsule_dealloc(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)o; + PyCapsule *capsule = (PyCapsule *)op; + PyObject_GC_UnTrack(op); if (capsule->destructor) { - capsule->destructor(o); + capsule->destructor(op); } - PyObject_Free(o); + PyObject_GC_Del(op); } @@ -279,6 +303,29 @@ capsule_repr(PyObject *o) } +static int +capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) +{ + if (capsule->traverse_func) { + return capsule->traverse_func((PyObject*)capsule, visit, arg); + } + else { + return 0; + } +} + + +static int +capsule_clear(PyCapsule *capsule) +{ + if (capsule->clear_func) { + return capsule->clear_func((PyObject*)capsule); + } + else { + return 0; + } +} + PyDoc_STRVAR(PyCapsule_Type__doc__, "Capsule objects let you wrap a C \"void *\" pointer in a Python\n\ @@ -293,27 +340,14 @@ Python import mechanism to link to one another.\n\ PyTypeObject PyCapsule_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "PyCapsule", /*tp_name*/ - sizeof(PyCapsule), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - capsule_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - capsule_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - 0, /*tp_flags*/ - PyCapsule_Type__doc__ /*tp_doc*/ + .tp_name = "PyCapsule", + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_basicsize = sizeof(PyCapsule), + .tp_dealloc = capsule_dealloc, + .tp_repr = capsule_repr, + .tp_doc = PyCapsule_Type__doc__, + .tp_traverse = (traverseproc)capsule_traverse, + .tp_clear = (inquiry)capsule_clear, }; From 3107b453bcceb0e2d2590fef3612859f2f802d72 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 23 Aug 2023 15:36:19 -0700 Subject: [PATCH 286/598] gh-108253: Fix reads of uninitialized memory in funcobject.c (#108383) --- Objects/funcobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 2820e47bdf2e6b..648b660859c04d 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -131,7 +131,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - _PyFunction_SetVersion(op, 0); + op->func_version = 0; _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return op; @@ -207,7 +207,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - _PyFunction_SetVersion(op, 0); + op->func_version = 0; _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; From 7a6cc3eb66805e6515d5db5b79e58e2f76b550c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 00:52:48 +0200 Subject: [PATCH 287/598] test_peg_generator and test_freeze require cpu (#108386) The test_peg_generator and test_tools.test_freeze tests now require the 'cpu' resource. Skip these tests unless the 'cpu' resource is enabled (it is disabled by default). These tests are no longer skipped if Python is built with ASAN or MSAN sanitizer. --- Lib/test/test_peg_generator/__init__.py | 6 ++---- Lib/test/test_tools/__init__.py | 6 ------ Lib/test/test_tools/test_freeze.py | 3 +++ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index 87281eb5e03c7f..c23542e254c99f 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -4,10 +4,8 @@ from test.support import load_package_tests -if support.check_sanitizer(address=True, memory=True): - # gh-90791: Skip the test because it is too slow when Python is built - # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. - raise unittest.SkipTest("test too slow on ASAN/MSAN build") +# Creating a virtual environment and building C extensions is slow +support.requires('cpu') # Load all tests in package diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index dde5d84e9edc6b..c4395c7c0ad0c9 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -7,12 +7,6 @@ from test.support import import_helper -if support.check_sanitizer(address=True, memory=True): - # gh-90791: Skip the test because it is too slow when Python is built - # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. - raise unittest.SkipTest("test too slow on ASAN/MSAN build") - - if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 2ba36ca208f967..3e9a48b5bc6a89 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -18,6 +18,9 @@ class TestFreeze(unittest.TestCase): def test_freeze_simple_script(self): + # Building Python is slow + support.requires('cpu') + script = textwrap.dedent(""" import sys print('running...') From 174e9da0836844a2138cc8915dd305cb2cd7a583 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 04:44:58 +0200 Subject: [PATCH 288/598] gh-108388: regrtest splits test_asyncio package (#108393) Currently, test_asyncio package is only splitted into sub-tests when using command "./python -m test". With this change, it's also splitted when passing it on the command line: "./python -m test test_asyncio". Remove the concept of "STDTESTS". Python is now mature enough to not have to bother with that anymore. Removing STDTESTS simplify the code. --- Lib/test/libregrtest/main.py | 25 +++++++------- Lib/test/libregrtest/runtest.py | 58 ++++++++++++++++----------------- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 361189199d3820..3d290c849b43ed 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,8 +11,8 @@ import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, runtest, get_abs_module, is_failed, - STDTESTS, NOTTESTS, PROGRESS_MIN_TIME, + findtests, split_test_packages, runtest, get_abs_module, is_failed, + PROGRESS_MIN_TIME, Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted, ChildError, DidNotRun) from test.libregrtest.setup import setup_tests @@ -246,26 +246,23 @@ def find_tests(self, tests): # add default PGO tests if no tests are specified setup_pgo_tests(self.ns) - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() + exclude = set() if self.ns.exclude: for arg in self.ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) + exclude.add(arg) self.ns.args = [] - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if self.ns.testdir: - alltests = findtests(self.ns.testdir, list(), set()) - else: - alltests = findtests(self.ns.testdir, stdtests, nottests) + alltests = findtests(testdir=self.ns.testdir, exclude=exclude) if not self.ns.fromfile: - self.selected = self.tests or self.ns.args or alltests + self.selected = self.tests or self.ns.args + if self.selected: + self.selected = split_test_packages(self.selected) + else: + self.selected = alltests else: self.selected = self.tests + if self.ns.single: self.selected = self.selected[:1] try: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 61595277ed6d5a..e927079da47a85 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -125,24 +125,6 @@ def __str__(self) -> str: # the test is running in background PROGRESS_MIN_TIME = 30.0 # seconds -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - #If these test directories are encountered recurse into them and treat each # test_ .py or dir as a separate test module. This can increase parallelism. # Beware this can't generally be done for any directory with sub-tests as the @@ -166,22 +148,38 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""): +def findtests(*, testdir=None, exclude=(), + split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) - names = os.listdir(testdir) tests = [] - others = set(stdtests) | nottests - for name in names: + for name in os.listdir(testdir): mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and mod not in others: - if mod in split_test_dirs: - subdir = os.path.join(testdir, mod) - mod = f"{base_mod or 'test'}.{mod}" - tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) - elif ext in (".py", ""): - tests.append(f"{base_mod}.{mod}" if base_mod else mod) - return stdtests + sorted(tests) + if (not mod.startswith("test_")) or (mod in exclude): + continue + if mod in split_test_dirs: + subdir = os.path.join(testdir, mod) + mod = f"{base_mod or 'test'}.{mod}" + tests.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, base_mod=mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if base_mod else mod) + return sorted(tests) + + +def split_test_packages(tests, *, testdir=None, exclude=(), + split_test_dirs=SPLITTESTDIRS): + testdir = findtestdir(testdir) + splitted = [] + for name in tests: + if name in split_test_dirs: + subdir = os.path.join(testdir, name) + splitted.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=name)) + else: + splitted.append(name) + return splitted def get_abs_module(ns: Namespace, test_name: str) -> str: From aa9a359ca2663195b0f04eef46109c28c4ff74d3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 05:35:39 +0200 Subject: [PATCH 289/598] gh-108388: Split test_multiprocessing_spawn (#108396) Split test_multiprocessing_fork, test_multiprocessing_forkserver and test_multiprocessing_spawn into test packages. Each package is made of 4 sub-tests: processes, threads, manager and misc. It allows running more tests in parallel and so reduce the total test duration. --- Lib/test/_test_multiprocessing.py | 10 +++++++++- Lib/test/libregrtest/runtest.py | 3 +++ .../__init__.py} | 11 ++++------- Lib/test/test_multiprocessing_fork/test_manager.py | 7 +++++++ Lib/test/test_multiprocessing_fork/test_misc.py | 7 +++++++ Lib/test/test_multiprocessing_fork/test_processes.py | 7 +++++++ Lib/test/test_multiprocessing_fork/test_threads.py | 7 +++++++ .../__init__.py} | 11 ++++------- .../test_multiprocessing_forkserver/test_manager.py | 7 +++++++ .../test_multiprocessing_forkserver/test_misc.py | 7 +++++++ .../test_processes.py | 7 +++++++ .../test_multiprocessing_forkserver/test_threads.py | 7 +++++++ Lib/test/test_multiprocessing_spawn.py | 12 ------------ Lib/test/test_multiprocessing_spawn/__init__.py | 9 +++++++++ Lib/test/test_multiprocessing_spawn/test_manager.py | 7 +++++++ Lib/test/test_multiprocessing_spawn/test_misc.py | 7 +++++++ .../test_multiprocessing_spawn/test_processes.py | 7 +++++++ Lib/test/test_multiprocessing_spawn/test_threads.py | 7 +++++++ .../2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst | 4 ++++ 19 files changed, 117 insertions(+), 27 deletions(-) rename Lib/test/{test_multiprocessing_fork.py => test_multiprocessing_fork/__init__.py} (66%) create mode 100644 Lib/test/test_multiprocessing_fork/test_manager.py create mode 100644 Lib/test/test_multiprocessing_fork/test_misc.py create mode 100644 Lib/test/test_multiprocessing_fork/test_processes.py create mode 100644 Lib/test/test_multiprocessing_fork/test_threads.py rename Lib/test/{test_multiprocessing_forkserver.py => test_multiprocessing_forkserver/__init__.py} (58%) create mode 100644 Lib/test/test_multiprocessing_forkserver/test_manager.py create mode 100644 Lib/test/test_multiprocessing_forkserver/test_misc.py create mode 100644 Lib/test/test_multiprocessing_forkserver/test_processes.py create mode 100644 Lib/test/test_multiprocessing_forkserver/test_threads.py delete mode 100644 Lib/test/test_multiprocessing_spawn.py create mode 100644 Lib/test/test_multiprocessing_spawn/__init__.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_manager.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_misc.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_processes.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_threads.py create mode 100644 Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a826e31f5ee079..343e9dcf769e86 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6126,7 +6126,8 @@ class ThreadsMixin(BaseMixin): # Functions used to create test cases from the base ones in this module # -def install_tests_in_module_dict(remote_globs, start_method): +def install_tests_in_module_dict(remote_globs, start_method, + only_type=None, exclude_types=False): __module__ = remote_globs['__name__'] local_globs = globals() ALL_TYPES = {'processes', 'threads', 'manager'} @@ -6139,6 +6140,10 @@ def install_tests_in_module_dict(remote_globs, start_method): continue assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES for type_ in base.ALLOWED_TYPES: + if only_type and type_ != only_type: + continue + if exclude_types: + continue newname = 'With' + type_.capitalize() + name[1:] Mixin = local_globs[type_.capitalize() + 'Mixin'] class Temp(base, Mixin, unittest.TestCase): @@ -6149,6 +6154,9 @@ class Temp(base, Mixin, unittest.TestCase): Temp.__module__ = __module__ remote_globs[newname] = Temp elif issubclass(base, unittest.TestCase): + if only_type: + continue + class Temp(base, object): pass Temp.__name__ = Temp.__qualname__ = name diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e927079da47a85..cddb37c87aa4d4 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -132,6 +132,9 @@ def __str__(self) -> str: SPLITTESTDIRS = { "test_asyncio", + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", } # Storage of uncollectable objects diff --git a/Lib/test/test_multiprocessing_fork.py b/Lib/test/test_multiprocessing_fork/__init__.py similarity index 66% rename from Lib/test/test_multiprocessing_fork.py rename to Lib/test/test_multiprocessing_fork/__init__.py index 5000edb7c5c299..aa1fff50b28f5f 100644 --- a/Lib/test/test_multiprocessing_fork.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -1,7 +1,6 @@ -import unittest -import test._test_multiprocessing - +import os.path import sys +import unittest from test import support if support.PGO: @@ -13,7 +12,5 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") -test._test_multiprocessing.install_tests_in_module_dict(globals(), 'fork') - -if __name__ == '__main__': - unittest.main() +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_fork/test_manager.py b/Lib/test/test_multiprocessing_fork/test_manager.py new file mode 100644 index 00000000000000..9efbb83bbb73bf --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_misc.py b/Lib/test/test_multiprocessing_fork/test_misc.py new file mode 100644 index 00000000000000..891a494020c3a1 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_processes.py b/Lib/test/test_multiprocessing_fork/test_processes.py new file mode 100644 index 00000000000000..e64e9afc0100e8 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_processes.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="processes") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_threads.py b/Lib/test/test_multiprocessing_fork/test_threads.py new file mode 100644 index 00000000000000..1670e34cb17f25 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver.py b/Lib/test/test_multiprocessing_forkserver/__init__.py similarity index 58% rename from Lib/test/test_multiprocessing_forkserver.py rename to Lib/test/test_multiprocessing_forkserver/__init__.py index 6ad5faf9e8a329..d91715a344dfa7 100644 --- a/Lib/test/test_multiprocessing_forkserver.py +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -1,7 +1,6 @@ -import unittest -import test._test_multiprocessing - +import os.path import sys +import unittest from test import support if support.PGO: @@ -10,7 +9,5 @@ if sys.platform == "win32": raise unittest.SkipTest("forkserver is not available on Windows") -test._test_multiprocessing.install_tests_in_module_dict(globals(), 'forkserver') - -if __name__ == '__main__': - unittest.main() +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_forkserver/test_manager.py b/Lib/test/test_multiprocessing_forkserver/test_manager.py new file mode 100644 index 00000000000000..14f8f10dfb4e2f --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_misc.py b/Lib/test/test_multiprocessing_forkserver/test_misc.py new file mode 100644 index 00000000000000..9cae1b50f71e00 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_processes.py b/Lib/test/test_multiprocessing_forkserver/test_processes.py new file mode 100644 index 00000000000000..360967cf1ae152 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_processes.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="processes") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_threads.py b/Lib/test/test_multiprocessing_forkserver/test_threads.py new file mode 100644 index 00000000000000..719c752aa05350 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn.py b/Lib/test/test_multiprocessing_spawn.py deleted file mode 100644 index 6558952308f25c..00000000000000 --- a/Lib/test/test_multiprocessing_spawn.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -import test._test_multiprocessing - -from test import support - -if support.PGO: - raise unittest.SkipTest("test is not helpful for PGO") - -test._test_multiprocessing.install_tests_in_module_dict(globals(), 'spawn') - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/__init__.py b/Lib/test/test_multiprocessing_spawn/__init__.py new file mode 100644 index 00000000000000..3fd0f9b390612a --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/__init__.py @@ -0,0 +1,9 @@ +import os.path +import unittest +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_spawn/test_manager.py b/Lib/test/test_multiprocessing_spawn/test_manager.py new file mode 100644 index 00000000000000..b40bea0bf61581 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_misc.py b/Lib/test/test_multiprocessing_spawn/test_misc.py new file mode 100644 index 00000000000000..32f37c5cc81ee1 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_processes.py b/Lib/test/test_multiprocessing_spawn/test_processes.py new file mode 100644 index 00000000000000..af764b0d8483ab --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_processes.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="processes") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_threads.py b/Lib/test/test_multiprocessing_spawn/test_threads.py new file mode 100644 index 00000000000000..c1257749b9c419 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst b/Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst new file mode 100644 index 00000000000000..8cf77b1cc187f1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst @@ -0,0 +1,4 @@ +Split test_multiprocessing_fork, test_multiprocessing_forkserver and +test_multiprocessing_spawn into test packages. Each package is made of 4 +sub-tests: processes, threads, manager and misc. It allows running more tests +in parallel and so reduce the total test duration. Patch by Victor Stinner. From 2eb60c1934f47671e6b3c9b90b6d9f1912d829a0 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Thu, 24 Aug 2023 04:23:01 -0400 Subject: [PATCH 290/598] gh-108111: Flush gzip write buffer before seeking, fixing bad writes (#108341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/gzip.py | 3 +++ Lib/test/test_gzip.py | 12 ++++++++++++ Misc/ACKS | 1 + .../2023-08-22-17-27-12.gh-issue-108111.N7a4u_.rst | 2 ++ 4 files changed, 18 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-22-17-27-12.gh-issue-108111.N7a4u_.rst diff --git a/Lib/gzip.py b/Lib/gzip.py index cf8b675064ce89..177f9080dc5af8 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -401,6 +401,9 @@ def seekable(self): def seek(self, offset, whence=io.SEEK_SET): if self.mode == WRITE: + self._check_not_closed() + # Flush buffer to ensure validity of self.offset + self._buffer.flush() if whence != io.SEEK_SET: if whence == io.SEEK_CUR: offset = self.offset + offset diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index b06b3b09411d62..128f933787a3f6 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -665,6 +665,18 @@ def flush(self, mode=-1): ] self.assertEqual(fc.modes, expected_modes) + def test_write_seek_write(self): + # Make sure that offset is up-to-date before seeking + # See issue GH-108111 + b = io.BytesIO() + message = b"important message here." + with gzip.GzipFile(fileobj=b, mode='w') as f: + f.write(message) + f.seek(len(message)) + f.write(message) + data = b.getvalue() + self.assertEqual(gzip.decompress(data), message * 2) + class TestOpen(BaseTest): def test_binary_modes(self): diff --git a/Misc/ACKS b/Misc/ACKS index 475c6ec3dab73b..70c4ea37adda75 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1150,6 +1150,7 @@ Colin Marc Vincent Marchetti David Marek Doug Marien +Chris Markiewicz Sven Marnach John Marshall Alex Martelli diff --git a/Misc/NEWS.d/next/Library/2023-08-22-17-27-12.gh-issue-108111.N7a4u_.rst b/Misc/NEWS.d/next/Library/2023-08-22-17-27-12.gh-issue-108111.N7a4u_.rst new file mode 100644 index 00000000000000..8eafa6cfbbf8cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-22-17-27-12.gh-issue-108111.N7a4u_.rst @@ -0,0 +1,2 @@ +Fix a regression introduced in GH-101251 for 3.12, resulting in an incorrect +offset calculation in :meth:`gzip.GzipFile.seek`. From 809ea7c4b6c2b818ae510f1f58e82b6b05ed4ef9 Mon Sep 17 00:00:00 2001 From: Daniele Procida Date: Thu, 24 Aug 2023 11:28:23 +0200 Subject: [PATCH 291/598] gh-107432 Update Porting Python 2 Code to Python 3 how-to (GH-107434) https://docs.python.org/3/howto/pyporting.html#porting-python-2-code-to-python-3 was written for another time. In this patch: - material that frames Python 3 as "new" is removed - descriptions and directions have been trimmed --- Doc/howto/pyporting.rst | 215 ++++++++++++++++++---------------------- 1 file changed, 94 insertions(+), 121 deletions(-) diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index baea3e85c3b84b..6c30a0dd7d6bcc 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -1,49 +1,47 @@ .. _pyporting-howto: -********************************* -Porting Python 2 Code to Python 3 -********************************* +************************************* +How to port Python 2 Code to Python 3 +************************************* :author: Brett Cannon .. topic:: Abstract - With Python 3 being the future of Python while Python 2 is still in active - use, it is good to have your project available for both major releases of - Python. This guide is meant to help you figure out how best to support both - Python 2 & 3 simultaneously. + Python 2 reached its official end-of-life at the start of 2020. This means + that no new bug reports, fixes, or changes will be made to Python 2 - it's + no longer supported. + + This guide is intended to provide you with a path to Python 3 for your + code, that includes compatibility with Python 2 as a first step. If you are looking to port an extension module instead of pure Python code, please see :ref:`cporting-howto`. - If you would like to read one core Python developer's take on why Python 3 - came into existence, you can read Nick Coghlan's `Python 3 Q & A`_ or - Brett Cannon's `Why Python 3 exists`_. - + The archived python-porting_ mailing list may contain some useful guidance. - For help with porting, you can view the archived python-porting_ mailing list. The Short Explanation ===================== -To make your project be single-source Python 2/3 compatible, the basic steps +To achieve Python 2/3 compatibility in a single code base, the basic steps are: #. Only worry about supporting Python 2.7 #. Make sure you have good test coverage (coverage.py_ can help; ``python -m pip install coverage``) -#. Learn the differences between Python 2 & 3 +#. Learn the differences between Python 2 and 3 #. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) #. Use Pylint_ to help make sure you don't regress on your Python 3 support (``python -m pip install pylint``) #. Use caniusepython3_ to find out which of your dependencies are blocking your use of Python 3 (``python -m pip install caniusepython3``) #. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 & 3 (tox_ can help test + to make sure you stay compatible with Python 2 and 3 (tox_ can help test against multiple versions of Python; ``python -m pip install tox``) #. Consider using optional static type checking to make sure your type usage - works in both Python 2 & 3 (e.g. use mypy_ to check your typing under both - Python 2 & Python 3; ``python -m pip install mypy``). + works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both + Python 2 and Python 3; ``python -m pip install mypy``). .. note:: @@ -55,43 +53,30 @@ are: Details ======= -A key point about supporting Python 2 & 3 simultaneously is that you can start -**today**! Even if your dependencies are not supporting Python 3 yet that does -not mean you can't modernize your code **now** to support Python 3. Most changes -required to support Python 3 lead to cleaner code using newer practices even in -Python 2 code. +Even if other factors - say, dependencies over which you have no control - +still require you to support Python 2, that does not prevent you taking the +step of including Python 3 support. -Another key point is that modernizing your Python 2 code to also support -Python 3 is largely automated for you. While you might have to make some API -decisions thanks to Python 3 clarifying text data versus binary data, the -lower-level work is now mostly done for you and thus can at least benefit from -the automated changes immediately. +Most changes required to support Python 3 lead to cleaner code using newer +practices even in Python 2 code. -Keep those key points in mind while you read on about the details of porting -your code to support Python 2 & 3 simultaneously. +Different versions of Python 2 +------------------------------ -Drop support for Python 2.6 and older -------------------------------------- +Ideally, your code should be compatible with Python 2.7, which was the +last supported version of Python 2. -While you can make Python 2.5 work with Python 3, it is **much** easier if you -only have to work with Python 2.7. If dropping Python 2.5 is not an -option then the six_ project can help you support Python 2.5 & 3 simultaneously -(``python -m pip install six``). Do realize, though, that nearly all the projects listed -in this HOWTO will not be available to you. +Some of the tools mentioned in this guide will not work with Python 2.6. -If you are able to skip Python 2.5 and older, then the required changes -to your code should continue to look and feel like idiomatic Python code. At -worst you will have to use a function instead of a method in some instances or -have to import a function instead of using a built-in one, but otherwise the -overall transformation should not feel foreign to you. +If absolutely necessary, the six_ project can help you support Python 2.5 and +3 simultaneously. Do realize, though, that nearly all the projects listed in +this guide will not be available to you. -But you should aim for only supporting Python 2.7. Python 2.6 is no longer -freely supported and thus is not receiving bugfixes. This means **you** will have -to work around any issues you come across with Python 2.6. There are also some -tools mentioned in this HOWTO which do not support Python 2.6 (e.g., Pylint_), -and this will become more commonplace as time goes on. It will simply be easier -for you if you only support the versions of Python that you have to support. +If you are able to skip Python 2.5 and older, the required changes to your +code will be minimal. At worst you will have to use a function instead of a +method in some instances or have to import a function instead of using a +built-in one. Make sure you specify the proper version support in your ``setup.py`` file @@ -118,62 +103,57 @@ coverage). If you don't already have a tool to measure test coverage then coverage.py_ is recommended. -Learn the differences between Python 2 & 3 -------------------------------------------- +Be aware of the differences between Python 2 and 3 +-------------------------------------------------- Once you have your code well-tested you are ready to begin porting your code to Python 3! But to fully understand how your code is going to change and what you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. Typically the two best ways of doing that -is reading the :ref:`"What's New" ` doc for each release of Python 3 and the -`Porting to Python 3`_ book (which is free online). There is also a handy -`cheat sheet`_ from the Python-Future project. +Python 3 makes in terms of Python 2. + +Some resources for understanding the differences and their implications for you +code: + +* the :ref:`"What's New" ` doc for each release of Python 3 +* the `Porting to Python 3`_ book (which is free online) +* the handy `cheat sheet`_ from the Python-Future project. Update your code ---------------- -Once you feel like you know what is different in Python 3 compared to Python 2, -it's time to update your code! You have a choice between two tools in porting -your code automatically: Futurize_ and Modernize_. Which tool you choose will -depend on how much like Python 3 you want your code to be. Futurize_ does its -best to make Python 3 idioms and practices exist in Python 2, e.g. backporting -the ``bytes`` type from Python 3 so that you have semantic parity between the -major versions of Python. Modernize_, -on the other hand, is more conservative and targets a Python 2/3 subset of -Python, directly relying on six_ to help provide compatibility. As Python 3 is -the future, it might be best to consider Futurize to begin adjusting to any new -practices that Python 3 introduces which you are not accustomed to yet. - -Regardless of which tool you choose, they will update your code to run under -Python 3 while staying compatible with the version of Python 2 you started with. -Depending on how conservative you want to be, you may want to run the tool over -your test suite first and visually inspect the diff to make sure the -transformation is accurate. After you have transformed your test suite and -verified that all the tests still pass as expected, then you can transform your -application code knowing that any tests which fail is a translation failure. +There are tools available that can port your code automatically. + +Futurize_ does its best to make Python 3 idioms and practices exist in Python +2, e.g. backporting the ``bytes`` type from Python 3 so that you have +semantic parity between the major versions of Python. This is the better +approach for most cases. + +Modernize_, on the other hand, is more conservative and targets a Python 2/3 +subset of Python, directly relying on six_ to help provide compatibility. + +A good approach is to run the tool over your test suite first and visually +inspect the diff to make sure the transformation is accurate. After you have +transformed your test suite and verified that all the tests still pass as +expected, then you can transform your application code knowing that any tests +which fail is a translation failure. Unfortunately the tools can't automate everything to make your code work under -Python 3 and so there are a handful of things you will need to update manually -to get full Python 3 support (which of these steps are necessary vary between -the tools). Read the documentation for the tool you choose to use to see what it -fixes by default and what it can do optionally to know what will (not) be fixed -for you and what you may have to fix on your own (e.g. using ``io.open()`` over -the built-in ``open()`` function is off by default in Modernize). Luckily, -though, there are only a couple of things to watch out for which can be -considered large issues that may be hard to debug if not watched for. +Python 3, and you will also need to read the tools' documentation in case some +options you need are turned off by default. +Key issues to be aware of and check for: Division ++++++++ -In Python 3, ``5 / 2 == 2.5`` and not ``2``; all division between ``int`` values -result in a ``float``. This change has actually been planned since Python 2.2 -which was released in 2002. Since then users have been encouraged to add -``from __future__ import division`` to any and all files which use the ``/`` and -``//`` operators or to be running the interpreter with the ``-Q`` flag. If you -have not been doing this then you will need to go through your code and do two -things: +In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all +division between ``int`` values result in a ``float``. This change has +actually been planned since Python 2.2 which was released in 2002. Since then +users have been encouraged to add ``from __future__ import division`` to any +and all files which use the ``/`` and ``//`` operators or to be running the +interpreter with the ``-Q`` flag. If you have not been doing this then you +will need to go through your code and do two things: #. Add ``from __future__ import division`` to your files #. Update any division operator as necessary to either use ``//`` to use floor @@ -197,30 +177,29 @@ specific type. This complicated the situation especially for anyone supporting multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` when they claimed text data support. -To make the distinction between text and binary data clearer and more -pronounced, Python 3 did what most languages created in the age of the internet -have done and made text and binary data distinct types that cannot blindly be -mixed together (Python predates widespread access to the internet). For any code -that deals only with text or only binary data, this separation doesn't pose an -issue. But for code that has to deal with both, it does mean you might have to -now care about when you are using text compared to binary data, which is why -this cannot be entirely automated. - -To start, you will need to decide which APIs take text and which take binary -(it is **highly** recommended you don't design APIs that can take both due to -the difficulty of keeping the code working; as stated earlier it is difficult to -do well). In Python 2 this means making sure the APIs that take text can work -with ``unicode`` and those that work with binary data work with the -``bytes`` type from Python 3 (which is a subset of ``str`` in Python 2 and acts -as an alias for ``bytes`` type in Python 2). Usually the biggest issue is -realizing which methods exist on which types in Python 2 & 3 simultaneously -(for text that's ``unicode`` in Python 2 and ``str`` in Python 3, for binary -that's ``str``/``bytes`` in Python 2 and ``bytes`` in Python 3). The following -table lists the **unique** methods of each data type across Python 2 & 3 -(e.g., the ``decode()`` method is usable on the equivalent binary data type in -either Python 2 or 3, but it can't be used by the textual data type consistently -between Python 2 and 3 because ``str`` in Python 3 doesn't have the method). Do -note that as of Python 3.5 the ``__mod__`` method was added to the bytes type. +Python 3 made text and binary data distinct types that cannot simply be mixed +together. For any code that deals only with text or only binary data, this +separation doesn't pose an issue. But for code that has to deal with both, it +does mean you might have to now care about when you are using text compared +to binary data, which is why this cannot be entirely automated. + +Decide which APIs take text and which take binary (it is **highly** recommended +you don't design APIs that can take both due to the difficulty of keeping the +code working; as stated earlier it is difficult to do well). In Python 2 this +means making sure the APIs that take text can work with ``unicode`` and those +that work with binary data work with the ``bytes`` type from Python 3 +(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes`` +type in Python 2). Usually the biggest issue is realizing which methods exist +on which types in Python 2 and 3 simultaneously (for text that's ``unicode`` +in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in +Python 2 and ``bytes`` in Python 3). + +The following table lists the **unique** methods of each data type across +Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary +data type in either Python 2 or 3, but it can't be used by the textual data +type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't +have the method). Do note that as of Python 3.5 the ``__mod__`` method was +added to the bytes type. ======================== ===================== **Text data** **Binary data** @@ -246,12 +225,11 @@ having to keep track of what type of data you are working with. The next issue is making sure you know whether the string literals in your code represent text or binary data. You should add a ``b`` prefix to any literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (there is a :mod:`__future__` import to force all unspecified +the text literal. (There is a :mod:`__future__` import to force all unspecified literals to be Unicode, but usage has shown it isn't as effective as adding a ``b`` or ``u`` prefix to all literals explicitly) -As part of this dichotomy you also need to be careful about opening files. -Unless you have been working on Windows, there is a chance you have not always +You also need to be careful about opening files. Possibly you have not always bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for binary reading). Under Python 3, binary files and text files are clearly distinct and mutually incompatible; see the :mod:`io` module for details. @@ -265,7 +243,7 @@ outdated practice of using :func:`codecs.open` as that's only necessary for keeping compatibility with Python 2.5. The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 & 3. Passing an integer to ``bytes`` in Python 2 +same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2 will give you the string representation of the integer: ``bytes(3) == '3'``. But in Python 3, an integer argument to ``bytes`` will give you a bytes object as long as the integer specified, filled with null bytes: @@ -400,7 +378,7 @@ Use continuous integration to stay compatible --------------------------------------------- Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 & 3. Probably the best tool for running +code always works under both Python 2 and 3. Probably the best tool for running your tests under multiple Python interpreters is tox_. You can then integrate tox with your continuous integration system so that you never accidentally break Python 2 or 3 support. @@ -413,11 +391,6 @@ separation of text/binary data handling or indexing on bytes you wouldn't easily find the mistake. This flag will raise an exception when these kinds of comparisons occur, making the mistake much easier to track down. -And that's mostly it! At this point your code base is compatible with both -Python 2 and 3 simultaneously. Your testing will also be set up so that you -don't accidentally break Python 2 or 3 compatibility regardless of which version -you typically run your tests under while developing. - Consider using optional static type checking -------------------------------------------- From c163d7f0b67a568e9b64eeb9c1cbbaa127818596 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Thu, 24 Aug 2023 20:22:50 +0800 Subject: [PATCH 292/598] gh-95855: Refactor platform triplet detection code, add detection for MIPS soft float and musl libc (#107221) - Move platform triplet detection code into Misc/platform_triplet.c - Refactor MIPS detection, use defined(__mips64) to detect MIPS64 - Compute libc values in separate section - Add detection for MIPS soft float - Add detection for musl musl supports SPE with its soft-float ABI: https://git.musl-libc.org/cgit/musl/commit/?id=7be59733d71ada3a32a98622507399253f1d5e48 Original patch by Christian Heimes. Co-authored-by: Christian Heimes Co-authored-by: Erlend E. Aasland --- ...3-07-25-02-30-00.gh-issue-95855.wA7rAf.rst | 2 + Misc/platform_triplet.c | 255 ++++++++++++++++++ configure | 192 +------------ configure.ac | 192 +------------ 4 files changed, 265 insertions(+), 376 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-07-25-02-30-00.gh-issue-95855.wA7rAf.rst create mode 100644 Misc/platform_triplet.c diff --git a/Misc/NEWS.d/next/Build/2023-07-25-02-30-00.gh-issue-95855.wA7rAf.rst b/Misc/NEWS.d/next/Build/2023-07-25-02-30-00.gh-issue-95855.wA7rAf.rst new file mode 100644 index 00000000000000..fdc8b33f1de5dc --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-07-25-02-30-00.gh-issue-95855.wA7rAf.rst @@ -0,0 +1,2 @@ +Refactor platform triplet detection code and add detection for MIPS soft +float and musl libc. diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c new file mode 100644 index 00000000000000..c5e64b93553a7c --- /dev/null +++ b/Misc/platform_triplet.c @@ -0,0 +1,255 @@ +/* Detect platform triplet from builtin defines + * cc -E Misc/platform_triplet.c | grep '^PLATFORM_TRIPLET=' | tr -d ' ' + */ +#undef bfin +#undef cris +#undef fr30 +#undef linux +#undef hppa +#undef hpux +#undef i386 +#undef mips +#undef powerpc +#undef sparc +#undef unix +#if defined(__ANDROID__) + # Android is not a multiarch system. +#elif defined(__linux__) +/* + * BEGIN of Linux block + */ +// Detect libc (based on config.guess) +# include +# if defined(__UCLIBC__) +# error uclibc not supported +# elif defined(__dietlibc__) +# error dietlibc not supported +# elif defined(__GLIBC__) +# define LIBC gnu +# define LIBC_X32 gnux32 +# if defined(__ARM_PCS_VFP) +# define LIBC_ARM gnueabihf +# else +# define LIBC_ARM gnueabi +# endif +# if defined(__loongarch__) +# if defined(__loongarch_soft_float) +# define LIBC_LA gnusf +# elif defined(__loongarch_single_float) +# define LIBC_LA gnuf32 +# elif defined(__loongarch_double_float) +# define LIBC_LA gnu +# else +# error unknown loongarch floating-point base abi +# endif +# endif +# if defined(_MIPS_SIM) +# if defined(__mips_hard_float) +# if _MIPS_SIM == _ABIO32 +# define LIBC_MIPS gnu +# elif _MIPS_SIM == _ABIN32 +# define LIBC_MIPS gnuabin32 +# elif _MIPS_SIM == _ABI64 +# define LIBC_MIPS gnuabi64 +# else +# error unknown mips sim value +# endif +# else +# if _MIPS_SIM == _ABIO32 +# define LIBC_MIPS gnusf +# elif _MIPS_SIM == _ABIN32 +# define LIBC_MIPS gnuabin32sf +# elif _MIPS_SIM == _ABI64 +# define LIBC_MIPS gnuabi64sf +# else +# error unknown mips sim value +# endif +# endif +# endif +# if defined(__SPE__) +# define LIBC_PPC gnuspe +# else +# define LIBC_PPC gnu +# endif +# else +// Heuristic to detect musl libc +# include +# ifdef __DEFINED_va_list +# define LIBC musl +# define LIBC_X32 muslx32 +# if defined(__ARM_PCS_VFP) +# define LIBC_ARM musleabihf +# else +# define LIBC_ARM musleabi +# endif +# if defined(__loongarch__) +# if defined(__loongarch_soft_float) +# define LIBC_LA muslsf +# elif defined(__loongarch_single_float) +# define LIBC_LA muslf32 +# elif defined(__loongarch_double_float) +# define LIBC_LA musl +# else +# error unknown loongarch floating-point base abi +# endif +# endif +# if defined(_MIPS_SIM) +# if defined(__mips_hard_float) +# if _MIPS_SIM == _ABIO32 +# define LIBC_MIPS musl +# elif _MIPS_SIM == _ABIN32 +# define LIBC_MIPS musln32 +# elif _MIPS_SIM == _ABI64 +# define LIBC_MIPS musl +# else +# error unknown mips sim value +# endif +# else +# if _MIPS_SIM == _ABIO32 +# define LIBC_MIPS muslsf +# elif _MIPS_SIM == _ABIN32 +# define LIBC_MIPS musln32sf +# elif _MIPS_SIM == _ABI64 +# define LIBC_MIPS muslsf +# else +# error unknown mips sim value +# endif +# endif +# endif +# if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) +# define LIBC_PPC muslsf +# else +# define LIBC_PPC musl +# endif +# else +# error unknown libc +# endif +# endif + +# if defined(__x86_64__) && defined(__LP64__) +PLATFORM_TRIPLET=x86_64-linux-LIBC +# elif defined(__x86_64__) && defined(__ILP32__) +PLATFORM_TRIPLET=x86_64-linux-LIBC_X32 +# elif defined(__i386__) +PLATFORM_TRIPLET=i386-linux-LIBC +# elif defined(__aarch64__) && defined(__AARCH64EL__) +# if defined(__ILP32__) +PLATFORM_TRIPLET=aarch64_ilp32-linux-LIBC +# else +PLATFORM_TRIPLET=aarch64-linux-LIBC +# endif +# elif defined(__aarch64__) && defined(__AARCH64EB__) +# if defined(__ILP32__) +PLATFORM_TRIPLET=aarch64_be_ilp32-linux-LIBC +# else +PLATFORM_TRIPLET=aarch64_be-linux-LIBC +# endif +# elif defined(__alpha__) +PLATFORM_TRIPLET=alpha-linux-LIBC +# elif defined(__ARM_EABI__) +# if defined(__ARMEL__) +PLATFORM_TRIPLET=arm-linux-LIBC_ARM +# else +PLATFORM_TRIPLET=armeb-linux-LIBC_ARM +# endif +# elif defined(__hppa__) +PLATFORM_TRIPLET=hppa-linux-LIBC +# elif defined(__ia64__) +PLATFORM_TRIPLET=ia64-linux-LIBC +# elif defined(__loongarch__) && defined(__loongarch_lp64) +PLATFORM_TRIPLET=loongarch64-linux-LIBC_LA +# elif defined(__m68k__) && !defined(__mcoldfire__) +PLATFORM_TRIPLET=m68k-linux-LIBC +# elif defined(__mips__) +# if defined(__mips_isa_rev) && (__mips_isa_rev >=6) +# if defined(_MIPSEL) && defined(__mips64) +PLATFORM_TRIPLET=mipsisa64r6el-linux-LIBC_MIPS +# elif defined(_MIPSEL) +PLATFORM_TRIPLET=mipsisa32r6el-linux-LIBC_MIPS +# elif defined(__mips64) +PLATFORM_TRIPLET=mipsisa64r6-linux-LIBC_MIPS +# else +PLATFORM_TRIPLET=mipsisa32r6-linux-LIBC_MIPS +# endif +# else +# if defined(_MIPSEL) && defined(__mips64) +PLATFORM_TRIPLET=mips64el-linux-LIBC_MIPS +# elif defined(_MIPSEL) +PLATFORM_TRIPLET=mipsel-linux-LIBC_MIPS +# elif defined(__mips64) +PLATFORM_TRIPLET=mips64-linux-LIBC_MIPS +# else +PLATFORM_TRIPLET=mips-linux-LIBC_MIPS +# endif +# endif +# elif defined(__or1k__) +PLATFORM_TRIPLET=or1k-linux-LIBC +# elif defined(__powerpc64__) +# if defined(__LITTLE_ENDIAN__) +PLATFORM_TRIPLET=powerpc64le-linux-LIBC +# else +PLATFORM_TRIPLET=powerpc64-linux-LIBC +# endif +# elif defined(__powerpc__) +PLATFORM_TRIPLET=powerpc-linux-LIBC_PPC +# elif defined(__s390x__) +PLATFORM_TRIPLET=s390x-linux-LIBC +# elif defined(__s390__) +PLATFORM_TRIPLET=s390-linux-LIBC +# elif defined(__sh__) && defined(__LITTLE_ENDIAN__) +PLATFORM_TRIPLET=sh4-linux-LIBC +# elif defined(__sparc__) && defined(__arch64__) +PLATFORM_TRIPLET=sparc64-linux-LIBC +# elif defined(__sparc__) +PLATFORM_TRIPLET=sparc-linux-LIBC +# elif defined(__riscv) +# if __riscv_xlen == 32 +PLATFORM_TRIPLET=riscv32-linux-LIBC +# elif __riscv_xlen == 64 +PLATFORM_TRIPLET=riscv64-linux-LIBC +# else +# error unknown platform triplet +# endif +# else +# error unknown platform triplet +# endif +/* + * END of Linux block + */ +#elif defined(__FreeBSD_kernel__) +# if defined(__LP64__) +PLATFORM_TRIPLET=x86_64-kfreebsd-gnu +# elif defined(__i386__) +PLATFORM_TRIPLET=i386-kfreebsd-gnu +# else +# error unknown platform triplet +# endif +#elif defined(__gnu_hurd__) +PLATFORM_TRIPLET=i386-gnu +#elif defined(__APPLE__) +PLATFORM_TRIPLET=darwin +#elif defined(__VXWORKS__) +PLATFORM_TRIPLET=vxworks +#elif defined(__wasm32__) +# if defined(__EMSCRIPTEN__) +PLATFORM_TRIPLET=wasm32-emscripten +# elif defined(__wasi__) +# if defined(_REENTRANT) +PLATFORM_TRIPLET=wasm32-wasi-threads +# else +PLATFORM_TRIPLET=wasm32-wasi +# endif +# else +# error unknown wasm32 platform +# endif +#elif defined(__wasm64__) +# if defined(__EMSCRIPTEN__) +PLATFORM_TRIPLET=wasm64-emscripten +# elif defined(__wasi__) +PLATFORM_TRIPLET=wasm64-wasi +# else +# error unknown wasm64 platform +# endif +#else +# error unknown platform triplet +#endif diff --git a/configure b/configure index f78b45a2c72dd2..34e874c49cce0a 100755 --- a/configure +++ b/configure @@ -6719,200 +6719,16 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the platform triplet based on compiler characteristics" >&5 printf %s "checking for the platform triplet based on compiler characteristics... " >&6; } -cat > conftest.c <=6) && defined(_MIPSEL) -# if _MIPS_SIM == _ABIO32 - mipsisa32r6el-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mipsisa64r6el-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mipsisa64r6el-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__mips_hard_float) && defined(__mips_isa_rev) && (__mips_isa_rev >=6) -# if _MIPS_SIM == _ABIO32 - mipsisa32r6-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mipsisa64r6-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mipsisa64r6-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__mips_hard_float) && defined(_MIPSEL) -# if _MIPS_SIM == _ABIO32 - mipsel-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mips64el-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mips64el-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__mips_hard_float) -# if _MIPS_SIM == _ABIO32 - mips-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mips64-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mips64-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__or1k__) - or1k-linux-gnu -# elif defined(__powerpc__) && defined(__SPE__) - powerpc-linux-gnuspe -# elif defined(__powerpc64__) -# if defined(__LITTLE_ENDIAN__) - powerpc64le-linux-gnu -# else - powerpc64-linux-gnu -# endif -# elif defined(__powerpc__) - powerpc-linux-gnu -# elif defined(__s390x__) - s390x-linux-gnu -# elif defined(__s390__) - s390-linux-gnu -# elif defined(__sh__) && defined(__LITTLE_ENDIAN__) - sh4-linux-gnu -# elif defined(__sparc__) && defined(__arch64__) - sparc64-linux-gnu -# elif defined(__sparc__) - sparc-linux-gnu -# elif defined(__riscv) -# if __riscv_xlen == 32 - riscv32-linux-gnu -# elif __riscv_xlen == 64 - riscv64-linux-gnu -# else -# error unknown platform triplet -# endif -# else -# error unknown platform triplet -# endif -#elif defined(__FreeBSD_kernel__) -# if defined(__LP64__) - x86_64-kfreebsd-gnu -# elif defined(__i386__) - i386-kfreebsd-gnu -# else -# error unknown platform triplet -# endif -#elif defined(__gnu_hurd__) - i386-gnu -#elif defined(__APPLE__) - darwin -#elif defined(__VXWORKS__) - vxworks -#elif defined(__wasm32__) -# if defined(__EMSCRIPTEN__) - wasm32-emscripten -# elif defined(__wasi__) -# if defined(_REENTRANT) - wasm32-wasi-threads -# else - wasm32-wasi -# endif -# else -# error unknown wasm32 platform -# endif -#elif defined(__wasm64__) -# if defined(__EMSCRIPTEN__) - wasm64-emscripten -# elif defined(__wasi__) - wasm64-wasi -# else -# error unknown wasm64 platform -# endif -#else -# error unknown platform triplet -#endif - -EOF - -if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then - PLATFORM_TRIPLET=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` - case "$build_os" in - linux-musl*) - PLATFORM_TRIPLET=`echo "$PLATFORM_TRIPLET" | sed 's/linux-gnu/linux-musl/'` - ;; - esac +if $CPP $CPPFLAGS $srcdir/Misc/platform_triplet.c >conftest.out 2>/dev/null; then + PLATFORM_TRIPLET=`grep '^PLATFORM_TRIPLET=' conftest.out | tr -d ' '` + PLATFORM_TRIPLET="${PLATFORM_TRIPLET#PLATFORM_TRIPLET=}" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PLATFORM_TRIPLET" >&5 printf "%s\n" "$PLATFORM_TRIPLET" >&6; } else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none" >&5 printf "%s\n" "none" >&6; } fi -rm -f conftest.c conftest.out +rm -f conftest.out { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for multiarch" >&5 printf %s "checking for multiarch... " >&6; } diff --git a/configure.ac b/configure.ac index fd18a452f2b610..4b343b5d531c4a 100644 --- a/configure.ac +++ b/configure.ac @@ -921,198 +921,14 @@ fi AC_MSG_CHECKING([for the platform triplet based on compiler characteristics]) -cat > conftest.c <=6) && defined(_MIPSEL) -# if _MIPS_SIM == _ABIO32 - mipsisa32r6el-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mipsisa64r6el-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mipsisa64r6el-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__mips_hard_float) && defined(__mips_isa_rev) && (__mips_isa_rev >=6) -# if _MIPS_SIM == _ABIO32 - mipsisa32r6-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mipsisa64r6-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mipsisa64r6-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__mips_hard_float) && defined(_MIPSEL) -# if _MIPS_SIM == _ABIO32 - mipsel-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mips64el-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mips64el-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__mips_hard_float) -# if _MIPS_SIM == _ABIO32 - mips-linux-gnu -# elif _MIPS_SIM == _ABIN32 - mips64-linux-gnuabin32 -# elif _MIPS_SIM == _ABI64 - mips64-linux-gnuabi64 -# else -# error unknown platform triplet -# endif -# elif defined(__or1k__) - or1k-linux-gnu -# elif defined(__powerpc__) && defined(__SPE__) - powerpc-linux-gnuspe -# elif defined(__powerpc64__) -# if defined(__LITTLE_ENDIAN__) - powerpc64le-linux-gnu -# else - powerpc64-linux-gnu -# endif -# elif defined(__powerpc__) - powerpc-linux-gnu -# elif defined(__s390x__) - s390x-linux-gnu -# elif defined(__s390__) - s390-linux-gnu -# elif defined(__sh__) && defined(__LITTLE_ENDIAN__) - sh4-linux-gnu -# elif defined(__sparc__) && defined(__arch64__) - sparc64-linux-gnu -# elif defined(__sparc__) - sparc-linux-gnu -# elif defined(__riscv) -# if __riscv_xlen == 32 - riscv32-linux-gnu -# elif __riscv_xlen == 64 - riscv64-linux-gnu -# else -# error unknown platform triplet -# endif -# else -# error unknown platform triplet -# endif -#elif defined(__FreeBSD_kernel__) -# if defined(__LP64__) - x86_64-kfreebsd-gnu -# elif defined(__i386__) - i386-kfreebsd-gnu -# else -# error unknown platform triplet -# endif -#elif defined(__gnu_hurd__) - i386-gnu -#elif defined(__APPLE__) - darwin -#elif defined(__VXWORKS__) - vxworks -#elif defined(__wasm32__) -# if defined(__EMSCRIPTEN__) - wasm32-emscripten -# elif defined(__wasi__) -# if defined(_REENTRANT) - wasm32-wasi-threads -# else - wasm32-wasi -# endif -# else -# error unknown wasm32 platform -# endif -#elif defined(__wasm64__) -# if defined(__EMSCRIPTEN__) - wasm64-emscripten -# elif defined(__wasi__) - wasm64-wasi -# else -# error unknown wasm64 platform -# endif -#else -# error unknown platform triplet -#endif - -EOF - -if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then - PLATFORM_TRIPLET=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` - case "$build_os" in - linux-musl*) - PLATFORM_TRIPLET=`echo "$PLATFORM_TRIPLET" | sed 's/linux-gnu/linux-musl/'` - ;; - esac +if $CPP $CPPFLAGS $srcdir/Misc/platform_triplet.c >conftest.out 2>/dev/null; then + PLATFORM_TRIPLET=`grep '^PLATFORM_TRIPLET=' conftest.out | tr -d ' '` + PLATFORM_TRIPLET="${PLATFORM_TRIPLET@%:@PLATFORM_TRIPLET=}" AC_MSG_RESULT([$PLATFORM_TRIPLET]) else AC_MSG_RESULT([none]) fi -rm -f conftest.c conftest.out +rm -f conftest.out AC_MSG_CHECKING([for multiarch]) AS_CASE([$ac_sys_system], From 67266266469fe0e817736227f39537182534c1a5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 15:59:12 +0200 Subject: [PATCH 293/598] gh-108314: Add PyDict_ContainsString() function (#108323) --- Doc/c-api/dict.rst | 20 ++++++++--- Doc/whatsnew/3.13.rst | 5 +++ Include/cpython/dictobject.h | 1 + Lib/test/test_capi/test_dict.py | 24 ++++++++++--- ...-08-22-18-45-20.gh-issue-108314.nOlmwq.rst | 4 +++ Modules/_ctypes/_ctypes.c | 34 +++++++++++-------- Modules/_testcapi/dict.c | 14 ++++++++ Objects/dictobject.c | 12 +++++++ Python/pylifecycle.c | 9 ++--- Python/pythonrun.c | 23 ++++++++----- 10 files changed, 111 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-08-22-18-45-20.gh-issue-108314.nOlmwq.rst diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e4c1d71a413a68..7810d05b0ebc2e 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -55,6 +55,15 @@ Dictionary Objects This is equivalent to the Python expression ``key in p``. +.. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) + + This is the same as :c:func:`PyDict_Contains`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Copy(PyObject *p) Return a new dictionary that contains the same key-value pairs as *p*. @@ -73,7 +82,7 @@ Dictionary Objects .. index:: single: PyUnicode_FromString() Insert *val* into the dictionary *p* using *key* as a key. *key* should - be a :c:expr:`const char*`. The key object is created using + be a :c:expr:`const char*` UTF-8 encoded bytes string. The key object is created using ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on failure. This function *does not* steal a reference to *val*. @@ -88,7 +97,8 @@ Dictionary Objects .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - Remove the entry in dictionary *p* which has a key specified by the string *key*. + Remove the entry in dictionary *p* which has a key specified by the UTF-8 + encoded bytes string *key*. If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. @@ -136,7 +146,8 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. .. note:: @@ -150,7 +161,8 @@ Dictionary Objects .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a - :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8509e18a7d792e..25eb5e981c55e1 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -862,6 +862,11 @@ New Features not needed. (Contributed by Victor Stinner in :gh:`106004`.) +* Added :c:func:`PyDict_ContainsString` function: same as + :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` + UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + (Contributed by Victor Stinner in :gh:`108314`.) + * Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is :term:`shutting down `. (Contributed by Victor Stinner in :gh:`108014`.) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 470f59436748fd..f44880991f4136 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -55,6 +55,7 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) +PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); PyAPI_FUNC(int) _PyDict_ContainsId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 9da6efd695ecab..b22fa20e14dfea 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -8,6 +8,7 @@ NULL = None +INVALID_UTF8 = b'\xff' class DictSubclass(dict): def __getitem__(self, key): @@ -137,7 +138,7 @@ def test_dict_getitemstring(self): self.assertEqual(getitemstring(dct2, b'a'), 1) self.assertIs(getitemstring(dct2, b'b'), KeyError) - self.assertIs(getitemstring({}, b'\xff'), KeyError) + self.assertIs(getitemstring({}, INVALID_UTF8), KeyError) self.assertIs(getitemstring(42, b'a'), KeyError) self.assertIs(getitemstring([], b'a'), KeyError) # CRASHES getitemstring({}, NULL) @@ -173,7 +174,7 @@ def test_dict_getitemstringref(self): self.assertIs(getitemstring(dct2, b'b'), KeyError) self.assertRaises(SystemError, getitemstring, 42, b'a') - self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, INVALID_UTF8) self.assertRaises(SystemError, getitemstring, [], b'a') # CRASHES getitemstring({}, NULL) # CRASHES getitemstring(NULL, b'a') @@ -213,6 +214,21 @@ def test_dict_contains(self): # CRASHES contains(42, 'a') # CRASHES contains(NULL, 'a') + def test_dict_contains_string(self): + contains_string = _testcapi.dict_containsstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(contains_string(dct, b'a')) + self.assertFalse(contains_string(dct, b'b')) + self.assertTrue(contains_string(dct, '\U0001f40d'.encode())) + self.assertRaises(UnicodeDecodeError, contains_string, dct, INVALID_UTF8) + + dct2 = DictSubclass(dct) + self.assertTrue(contains_string(dct2, b'a')) + self.assertFalse(contains_string(dct2, b'b')) + + # CRASHES contains({}, NULL) + # CRASHES contains(NULL, b'a') + def test_dict_setitem(self): setitem = _testcapi.dict_setitem dct = {} @@ -245,7 +261,7 @@ def test_dict_setitemstring(self): setitemstring(dct2, b'a', 5) self.assertEqual(dct2, {'a': 5}) - self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(UnicodeDecodeError, setitemstring, {}, INVALID_UTF8, 5) self.assertRaises(SystemError, setitemstring, UserDict(), b'a', 5) self.assertRaises(SystemError, setitemstring, 42, b'a', 5) # CRASHES setitemstring({}, NULL, 5) @@ -287,7 +303,7 @@ def test_dict_delitemstring(self): self.assertEqual(dct2, {'c': 2}) self.assertRaises(KeyError, delitemstring, dct2, b'b') - self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(UnicodeDecodeError, delitemstring, {}, INVALID_UTF8) self.assertRaises(SystemError, delitemstring, UserDict({'a': 1}), b'a') self.assertRaises(SystemError, delitemstring, 42, b'a') # CRASHES delitemstring({}, NULL) diff --git a/Misc/NEWS.d/next/C API/2023-08-22-18-45-20.gh-issue-108314.nOlmwq.rst b/Misc/NEWS.d/next/C API/2023-08-22-18-45-20.gh-issue-108314.nOlmwq.rst new file mode 100644 index 00000000000000..90ae50a291b937 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-22-18-45-20.gh-issue-108314.nOlmwq.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyDict_ContainsString` function: same as +:c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` +UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. +Patch by Victor Stinner. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index dc80291d3b810b..ed9efcad9ab0c8 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5139,8 +5139,6 @@ static PyObject * Pointer_get_contents(CDataObject *self, void *closure) { StgDictObject *stgdict; - PyObject *keep, *ptr_probe; - CDataObject *ptr2ptr; if (*(void **)self->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, @@ -5151,29 +5149,37 @@ Pointer_get_contents(CDataObject *self, void *closure) stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for pointer instances */ - keep = GetKeepedObjects(self); + PyObject *keep = GetKeepedObjects(self); if (keep != NULL) { // check if it's a pointer to a pointer: // pointers will have '0' key in the _objects - ptr_probe = PyDict_GetItemString(keep, "0"); - - if (ptr_probe != NULL) { - ptr2ptr = (CDataObject*) PyDict_GetItemString(keep, "1"); - if (ptr2ptr == NULL) { + int ptr_probe = PyDict_ContainsString(keep, "0"); + if (ptr_probe < 0) { + return NULL; + } + if (ptr_probe) { + PyObject *item; + if (PyDict_GetItemStringRef(keep, "1", &item) < 0) { + return NULL; + } + if (item == NULL) { PyErr_SetString(PyExc_ValueError, - "Unexpected NULL pointer in _objects"); + "Unexpected NULL pointer in _objects"); return NULL; } - // don't construct a new object, - // return existing one instead to preserve refcount +#ifndef NDEBUG + CDataObject *ptr2ptr = (CDataObject *)item; + // Don't construct a new object, + // return existing one instead to preserve refcount. + // Double-check that we are returning the same thing. assert( *(void**) self->b_ptr == ptr2ptr->b_ptr || *(void**) self->b_value.c == ptr2ptr->b_ptr || *(void**) self->b_ptr == ptr2ptr->b_value.c || *(void**) self->b_value.c == ptr2ptr->b_value.c - ); // double-check that we are returning the same thing - Py_INCREF(ptr2ptr); - return (PyObject *) ptr2ptr; + ); +#endif + return item; } } diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index b1dfcf4c707da7..6c3f9cd22faa5a 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -74,6 +74,19 @@ dict_contains(PyObject *self, PyObject *args) RETURN_INT(PyDict_Contains(obj, key)); } +static PyObject * +dict_containsstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &key, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyDict_ContainsString(obj, key)); +} + static PyObject * dict_size(PyObject *self, PyObject *obj) { @@ -349,6 +362,7 @@ static PyMethodDef test_methods[] = { {"dict_getitemref", dict_getitemref, METH_VARARGS}, {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_setitem", dict_setitem, METH_VARARGS}, {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, {"dict_delitem", dict_delitem, METH_VARARGS}, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f9701f6b4b09ad..10205e379a569b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3741,6 +3741,18 @@ PyDict_Contains(PyObject *op, PyObject *key) return (ix != DKIX_EMPTY && value != NULL); } +int +PyDict_ContainsString(PyObject *op, const char *key) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + return -1; + } + int res = PyDict_Contains(op, key_obj); + Py_DECREF(key_obj); + return res; +} + /* Internal version of PyDict_Contains used when the hash value is already known */ int _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 18614264989185..7d362af32cbc36 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2220,10 +2220,11 @@ add_main_module(PyInterpreterState *interp) } Py_DECREF(ann_dict); - if (_PyDict_GetItemStringWithError(d, "__builtins__") == NULL) { - if (PyErr_Occurred()) { - return _PyStatus_ERR("Failed to test __main__.__builtins__"); - } + int has_builtins = PyDict_ContainsString(d, "__builtins__"); + if (has_builtins < 0) { + return _PyStatus_ERR("Failed to test __main__.__builtins__"); + } + if (!has_builtins) { PyObject *bimod = PyImport_ImportModule("builtins"); if (bimod == NULL) { return _PyStatus_ERR("Failed to retrieve builtins module"); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 05b7dfa61d1a52..0e118b03d6a111 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -413,10 +413,11 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, PyObject *dict = PyModule_GetDict(main_module); // borrowed ref int set_file_name = 0; - if (_PyDict_GetItemStringWithError(dict, "__file__") == NULL) { - if (PyErr_Occurred()) { - goto done; - } + int has_file = PyDict_ContainsString(dict, "__file__"); + if (has_file < 0) { + goto done; + } + if (!has_file) { if (PyDict_SetItemString(dict, "__file__", filename) < 0) { goto done; } @@ -1713,13 +1714,17 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py _PyRuntime.signals.unhandled_keyboard_interrupt = 0; /* Set globals['__builtins__'] if it doesn't exist */ - if (globals != NULL && _PyDict_GetItemStringWithError(globals, "__builtins__") == NULL) { - if (PyErr_Occurred() || - PyDict_SetItemString(globals, "__builtins__", - tstate->interp->builtins) < 0) - { + if (globals != NULL) { + int has_builtins = PyDict_ContainsString(globals, "__builtins__"); + if (has_builtins < 0) { return NULL; } + if (!has_builtins) { + if (PyDict_SetItemString(globals, "__builtins__", + tstate->interp->builtins) < 0) { + return NULL; + } + } } v = PyEval_EvalCode((PyObject*)co, globals, locals); From a35d48d4bd24c11760661a74962ca1f134094f41 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 16:37:59 +0200 Subject: [PATCH 294/598] gh-108240: _PyCapsule_SetTraverse() rejects NULL callbacks (#108417) --- Objects/capsule.c | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Objects/capsule.c b/Objects/capsule.c index aa320013c53460..a1abcf683f0133 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -1,6 +1,9 @@ /* Wrap void * pointers to be passed between C modules */ #include "Python.h" +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK() + /* Internal structure of PyCapsule */ typedef struct { @@ -71,7 +74,7 @@ PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) capsule->destructor = destructor; capsule->traverse_func = NULL; capsule->clear_func = NULL; - // Only track the capsule if _PyCapsule_SetTraverse() is called + // Only track the object by the GC when _PyCapsule_SetTraverse() is called return (PyObject *)capsule; } @@ -204,8 +207,14 @@ _PyCapsule_SetTraverse(PyObject *op, traverseproc traverse_func, inquiry clear_f } PyCapsule *capsule = (PyCapsule *)op; - if (!PyObject_GC_IsTracked(op)) { - PyObject_GC_Track(op); + if (traverse_func == NULL || clear_func == NULL) { + PyErr_SetString(PyExc_ValueError, + "_PyCapsule_SetTraverse() called with NULL callback"); + return -1; + } + + if (!_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_TRACK(op); } capsule->traverse_func = traverse_func; @@ -306,24 +315,22 @@ capsule_repr(PyObject *o) static int capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) { - if (capsule->traverse_func) { - return capsule->traverse_func((PyObject*)capsule, visit, arg); - } - else { - return 0; - } + // Capsule object is only tracked by the GC + // if _PyCapsule_SetTraverse() is called + assert(capsule->traverse_func != NULL); + + return capsule->traverse_func((PyObject*)capsule, visit, arg); } static int capsule_clear(PyCapsule *capsule) { - if (capsule->clear_func) { - return capsule->clear_func((PyObject*)capsule); - } - else { - return 0; - } + // Capsule object is only tracked by the GC + // if _PyCapsule_SetTraverse() is called + assert(capsule->clear_func != NULL); + + return capsule->clear_func((PyObject*)capsule); } From 3f7e93be51699fca6ee72c656278732b47cca833 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 17:17:40 +0200 Subject: [PATCH 295/598] gh-107211: No longer export PyTime internal functions (#108422) No longer export these 2 internal C API functions: * _PyTime_AsNanoseconds() * _PyTime_GetSystemClockWithInfo() Change comment style to "// comment" and add comment explaining why other functions have to be exported. --- Include/internal/pycore_time.h | 264 ++++++++++++++++++--------------- 1 file changed, 146 insertions(+), 118 deletions(-) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 318fe4b3a32239..d8ea63a7242ccc 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -66,9 +66,9 @@ struct _time_runtime_state { struct timeval; #endif -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ +// _PyTime_t: Python timestamp with subsecond precision. It can be used to +// store a duration, and so indirectly a date (related to another date, like +// UNIX epoch). typedef int64_t _PyTime_t; // _PyTime_MIN nanoseconds is around -292.3 years #define _PyTime_MIN INT64_MIN @@ -77,52 +77,61 @@ typedef int64_t _PyTime_t; #define _SIZEOF_PYTIME_T 8 typedef enum { - /* Round towards minus infinity (-inf). - For example, used to read a clock. */ + // Round towards minus infinity (-inf). + // For example, used to read a clock. _PyTime_ROUND_FLOOR=0, - /* Round towards infinity (+inf). - For example, used for timeout to wait "at least" N seconds. */ + + // Round towards infinity (+inf). + // For example, used for timeout to wait "at least" N seconds. _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going to nearest even integer. - For example, used to round from a Python float. */ + + // Round to nearest with ties going to nearest even integer. + // For example, used to round from a Python float. _PyTime_ROUND_HALF_EVEN=2, - /* Round away from zero - For example, used for timeout. _PyTime_ROUND_CEILING rounds - -1e-9 to 0 milliseconds which causes bpo-31786 issue. - _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps - the timeout sign as expected. select.poll(timeout) must block - for negative values." */ + + // Round away from zero + // For example, used for timeout. _PyTime_ROUND_CEILING rounds + // -1e-9 to 0 milliseconds which causes bpo-31786 issue. + // _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps + // the timeout sign as expected. select.poll(timeout) must block + // for negative values. _PyTime_ROUND_UP=3, - /* _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be - used for timeouts. */ + + // _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be + // used for timeouts. _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP } _PyTime_round_t; -/* Convert a time_t to a PyLong. */ +// Convert a time_t to a PyLong. +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyLong_FromTime_t(time_t sec); -/* Convert a PyLong to a time_t. */ +// Convert a PyLong to a time_t. +// Export for '_datetime' shared extension PyAPI_FUNC(time_t) _PyLong_AsTime_t(PyObject *obj); -/* Convert a number of seconds, int or float, to time_t. */ +// Convert a number of seconds, int or float, to time_t. +// Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_ObjectToTime_t( PyObject *obj, time_t *sec, _PyTime_round_t); -/* Convert a number of seconds, int or float, to a timeval structure. - usec is in the range [0; 999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000). */ +// Convert a number of seconds, int or float, to a timeval structure. +// usec is in the range [0; 999999] and rounded towards zero. +// For example, -1.2 is converted to (-2, 800000). +// Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_ObjectToTimeval( PyObject *obj, time_t *sec, long *usec, _PyTime_round_t); -/* Convert a number of seconds, int or float, to a timespec structure. - nsec is in the range [0; 999999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000000). */ +// Convert a number of seconds, int or float, to a timespec structure. +// nsec is in the range [0; 999999999] and rounded towards zero. +// For example, -1.2 is converted to (-2, 800000000). +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(int) _PyTime_ObjectToTimespec( PyObject *obj, time_t *sec, @@ -130,50 +139,58 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( _PyTime_round_t); -/* Create a timestamp from a number of seconds. */ +// Create a timestamp from a number of seconds. +// Export for '_socket' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); -/* Macro to create a timestamp from a number of seconds, no integer overflow. - Only use the macro for small values, prefer _PyTime_FromSeconds(). */ +// Macro to create a timestamp from a number of seconds, no integer overflow. +// Only use the macro for small values, prefer _PyTime_FromSeconds(). #define _PYTIME_FROMSECONDS(seconds) \ ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) -/* Create a timestamp from a number of nanoseconds. */ +// Create a timestamp from a number of nanoseconds. +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); -/* Create a timestamp from a number of microseconds. - * Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. */ +// Create a timestamp from a number of microseconds. +// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); -/* Create a timestamp from nanoseconds (Python int). */ +// Create a timestamp from nanoseconds (Python int). +// Export for '_lsprof' shared extension. PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, PyObject *obj); -/* Convert a number of seconds (Python float or int) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ +// Convert a number of seconds (Python float or int) to a timestamp. +// Raise an exception and return -1 on error, return 0 on success. +// Export for '_socket' shared extension. PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round); -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ +// Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. +// Raise an exception and return -1 on error, return 0 on success. +// Export for 'select' shared extension. PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round); -/* Convert a timestamp to a number of seconds as a C double. */ +// Convert a timestamp to a number of seconds as a C double. +// Export for '_socket' shared extension. PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); -/* Convert timestamp to a number of milliseconds (10^-3 seconds). */ +// Convert timestamp to a number of milliseconds (10^-3 seconds). +// Export for '_ssl' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, _PyTime_round_t round); -/* Convert timestamp to a number of microseconds (10^-6 seconds). */ +// Convert timestamp to a number of microseconds (10^-6 seconds). +// Export for '_queue' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round); -/* Convert timestamp to a number of nanoseconds (10^-9 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsNanoseconds(_PyTime_t t); +// Convert timestamp to a number of nanoseconds (10^-9 seconds). +extern _PyTime_t _PyTime_AsNanoseconds(_PyTime_t t); #ifdef MS_WINDOWS // Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). @@ -181,36 +198,40 @@ extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, _PyTime_round_t round); #endif -/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int - object. */ +// Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int +// object. +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); #ifndef MS_WINDOWS -/* Create a timestamp from a timeval structure. - Raise an exception and return -1 on overflow, return 0 on success. */ +// Create a timestamp from a timeval structure. +// Raise an exception and return -1 on overflow, return 0 on success. extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); #endif -/* Convert a timestamp to a timeval structure (microsecond resolution). - tv_usec is always positive. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ +// Convert a timestamp to a timeval structure (microsecond resolution). +// tv_usec is always positive. +// Raise an exception and return -1 if the conversion overflowed, +// return 0 on success. +// Export for 'select' shared extension. PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); -/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ +// Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. +// On overflow, clamp tv_sec to _PyTime_t min/max. +// Export for 'select' shared extension. PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); -/* Convert a timestamp to a number of seconds (secs) and microseconds (us). - us is always positive. This function is similar to _PyTime_AsTimeval() - except that secs is always a time_t type, whereas the timeval structure - uses a C long for tv_sec on Windows. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ +// Convert a timestamp to a number of seconds (secs) and microseconds (us). +// us is always positive. This function is similar to _PyTime_AsTimeval() +// except that secs is always a time_t type, whereas the timeval structure +// uses a C long for tv_sec on Windows. +// Raise an exception and return -1 if the conversion overflowed, +// return 0 on success. +// Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( _PyTime_t t, time_t *secs, @@ -218,17 +239,19 @@ PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( _PyTime_round_t round); #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) -/* Create a timestamp from a timespec structure. - Raise an exception and return -1 on overflow, return 0 on success. */ +// Create a timestamp from a timespec structure. +// Raise an exception and return -1 on overflow, return 0 on success. extern int _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); -/* Convert a timestamp to a timespec structure (nanosecond resolution). - tv_nsec is always positive. - Raise an exception and return -1 on error, return 0 on success. */ +// Convert a timestamp to a timespec structure (nanosecond resolution). +// tv_nsec is always positive. +// Raise an exception and return -1 on error, return 0 on success. +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); -/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ +// Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. +// On overflow, clamp tv_sec to _PyTime_t min/max. +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); #endif @@ -236,14 +259,14 @@ PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); // Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); -/* Compute ticks * mul / div. - Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. - The caller must ensure that ((div - 1) * mul) cannot overflow. */ +// Compute ticks * mul / div. +// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +// The caller must ensure that ((div - 1) * mul) cannot overflow. extern _PyTime_t _PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, _PyTime_t div); -/* Structure used by time.get_clock_info() */ +// Structure used by time.get_clock_info() typedef struct { const char *implementation; int monotonic; @@ -251,72 +274,77 @@ typedef struct { double resolution; } _Py_clock_info_t; -/* Get the current time from the system clock. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetSystemClockWithInfo() to check for failure. */ +// Get the current time from the system clock. +// +// If the internal clock fails, silently ignore the error and return 0. +// On integer overflow, silently ignore the overflow and clamp the clock to +// [_PyTime_MIN; _PyTime_MAX]. +// +// Use _PyTime_GetSystemClockWithInfo() to check for failure. +// Export for '_random' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); -/* Get the current time from the system clock. - * On success, set *t and *info (if not NULL), and return 0. - * On error, raise an exception and return -1. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( +// Get the current time from the system clock. +// On success, set *t and *info (if not NULL), and return 0. +// On error, raise an exception and return -1. +extern int _PyTime_GetSystemClockWithInfo( _PyTime_t *t, _Py_clock_info_t *info); -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ +// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. +// The clock is not affected by system clock updates. The reference point of +// the returned value is undefined, so that only the difference between the +// results of consecutive calls is valid. +// +// If the internal clock fails, silently ignore the error and return 0. +// On integer overflow, silently ignore the overflow and clamp the clock to +// [_PyTime_MIN; _PyTime_MAX]. +// +// Use _PyTime_GetMonotonicClockWithInfo() to check for failure. +// Export for '_random' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ +// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. +// The clock is not affected by system clock updates. The reference point of +// the returned value is undefined, so that only the difference between the +// results of consecutive calls is valid. +// +// Fill info (if set) with information of the function used to get the time. +// +// Return 0 on success, raise an exception and return -1 on error. +// Export for '_testsinglephase' shared extension. PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( _PyTime_t *t, _Py_clock_info_t *info); -/* Converts a timestamp to the Gregorian time, using the local time zone. - Return 0 on success, raise an exception and return -1 on error. */ +// Converts a timestamp to the Gregorian time, using the local time zone. +// Return 0 on success, raise an exception and return -1 on error. +// Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); -/* Converts a timestamp to the Gregorian time, assuming UTC. - Return 0 on success, raise an exception and return -1 on error. */ +// Converts a timestamp to the Gregorian time, assuming UTC. +// Return 0 on success, raise an exception and return -1 on error. +// Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ +// Get the performance counter: clock with the highest available resolution to +// measure a short duration. +// +// If the internal clock fails, silently ignore the error and return 0. +// On integer overflow, silently ignore the overflow and clamp the clock to +// [_PyTime_MIN; _PyTime_MAX]. +// +// Use _PyTime_GetPerfCounterWithInfo() to check for failure. +// Export for '_lsprof' shared extension. PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ +// Get the performance counter: clock with the highest available resolution to +// measure a short duration. +// +// Fill info (if set) with information of the function used to get the time. +// +// Return 0 on success, raise an exception and return -1 on error. extern int _PyTime_GetPerfCounterWithInfo( _PyTime_t *t, _Py_clock_info_t *info); @@ -336,4 +364,4 @@ PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_TIME_H */ +#endif // !Py_INTERNAL_TIME_H From ea871c9b0f08399e440baed95a3e5793d6e0ea66 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 17:28:35 +0200 Subject: [PATCH 296/598] gh-107211: No longer export internal functions (6) (#108424) No longer export these 5 internal C API functions: * _PyArena_AddPyObject() * _PyArena_Free() * _PyArena_Malloc() * _PyArena_New() * _Py_FatalRefcountErrorFunc() Change comment style to "// comment" and add comment explaining why other functions have to be exported. --- Include/internal/pycore_interp.h | 62 ++++++++++++++-------------- Include/internal/pycore_long.h | 1 - Include/internal/pycore_object.h | 6 +-- Include/internal/pycore_obmalloc.h | 2 +- Include/internal/pycore_pathconfig.h | 1 + Include/internal/pycore_pyarena.h | 8 ++-- Include/internal/pycore_pyerrors.h | 1 + Include/internal/pycore_pyhash.h | 7 +++- 8 files changed, 45 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index c21c352608d161..9a75b002e5bc1e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -243,40 +243,40 @@ extern void _PyInterpreterState_IDDecref(PyInterpreterState *); extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); -/* Get a copy of the current interpreter configuration. - - Return 0 on success. Raise an exception and return -1 on error. - - The caller must initialize 'config', using PyConfig_InitPythonConfig() - for example. - - Python must be preinitialized to call this method. - The caller must hold the GIL. - - Once done with the configuration, PyConfig_Clear() must be called to clear - it. - - Export for '_testinternalcapi' shared extension. */ +// Get a copy of the current interpreter configuration. +// +// Return 0 on success. Raise an exception and return -1 on error. +// +// The caller must initialize 'config', using PyConfig_InitPythonConfig() +// for example. +// +// Python must be preinitialized to call this method. +// The caller must hold the GIL. +// +// Once done with the configuration, PyConfig_Clear() must be called to clear +// it. +// +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( struct PyConfig *config); -/* Set the configuration of the current interpreter. - - This function should be called during or just after the Python - initialization. - - Update the sys module with the new configuration. If the sys module was - modified directly after the Python initialization, these changes are lost. - - Some configuration like faulthandler or warnoptions can be updated in the - configuration, but don't reconfigure Python (don't enable/disable - faulthandler and don't reconfigure warnings filters). - - Return 0 on success. Raise an exception and return -1 on error. - - The configuration should come from _PyInterpreterState_GetConfigCopy(). - - Export for '_testinternalcapi' shared extension. */ +// Set the configuration of the current interpreter. +// +// This function should be called during or just after the Python +// initialization. +// +// Update the sys module with the new configuration. If the sys module was +// modified directly after the Python initialization, these changes are lost. +// +// Some configuration like faulthandler or warnoptions can be updated in the +// configuration, but don't reconfigure Python (don't enable/disable +// faulthandler and don't reconfigure warnings filters). +// +// Return 0 on success. Raise an exception and return -1 on error. +// +// The configuration should come from _PyInterpreterState_GetConfigCopy(). +// +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(int) _PyInterpreterState_SetConfig( const struct PyConfig *config); diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3dc00ec7e04c6f..4eb70f7e3709dc 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -83,7 +83,6 @@ extern PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); extern PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); extern PyObject *_PyLong_Subtract(PyLongObject *left, PyLongObject *right); -// Used by _PyBytes_FromHex(), _PyBytes_DecodeEscape(), Python/mystrtoul.c. // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 5e2d13a5220556..c6007718bae5a4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -54,7 +54,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); .ob_size = size \ }, -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( +extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, const char *message); @@ -469,13 +469,11 @@ extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( (meth)((self), (args), (kw)) #endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE -// Export for '_pickle' shared extension +// Export these 2 symbols for '_pickle' shared extension PyAPI_DATA(PyTypeObject) _PyNone_Type; -// Export for '_pickle' shared extension PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; // Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. -// Defined in Objects/object.c. // Export for the stable ABI. PyAPI_DATA(int) _Py_SwappedOp[]; diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index ca2a0419b4f038..b0dbf53d4e3d15 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -687,7 +687,7 @@ extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *); #ifdef WITH_PYMALLOC -// Export the symbol for the 3rd party guppy3 project +// Export the symbol for the 3rd party 'guppy3' project PyAPI_FUNC(int) _PyObject_DebugMallocStats(FILE *out); #endif diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 729f3e09ce1ca4..a1ce1b19a00283 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -10,6 +10,7 @@ extern "C" { // Export for '_testinternalcapi' shared extension PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); + extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config); extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config); extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void); diff --git a/Include/internal/pycore_pyarena.h b/Include/internal/pycore_pyarena.h index 08262fba2daeee..6bd926a95cda83 100644 --- a/Include/internal/pycore_pyarena.h +++ b/Include/internal/pycore_pyarena.h @@ -37,8 +37,8 @@ typedef struct _arena PyArena; XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but XXX an exception is not set in that case). */ -PyAPI_FUNC(PyArena*) _PyArena_New(void); -PyAPI_FUNC(void) _PyArena_Free(PyArena *); +extern PyArena* _PyArena_New(void); +extern void _PyArena_Free(PyArena *); /* Mostly like malloc(), return the address of a block of memory spanning * `size` bytes, or return NULL (without setting an exception) if enough @@ -52,13 +52,13 @@ PyAPI_FUNC(void) _PyArena_Free(PyArena *); * until _PyArena_Free(ar) is called, at which point all pointers obtained * from the arena `ar` become invalid simultaneously. */ -PyAPI_FUNC(void*) _PyArena_Malloc(PyArena *, size_t size); +extern void* _PyArena_Malloc(PyArena *, size_t size); /* This routine isn't a proper arena allocation routine. It takes * a PyObject* and records it so that it can be DECREFed when the * arena is freed. */ -PyAPI_FUNC(int) _PyArena_AddPyObject(PyArena *, PyObject *); +extern int _PyArena_AddPyObject(PyArena *, PyObject *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 91fd68984523dd..5d2ad50e0d2cfd 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -152,6 +152,7 @@ extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); + // Export for '_testinternalcapi' shared extension PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index da9abd28b499a2..78bf0c7d07eb10 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -7,11 +7,14 @@ /* Helpers for hash functions */ extern Py_hash_t _Py_HashDouble(PyObject *, double); -// _decimal shared extensions uses _Py_HashPointer() + +// Export for '_decimal' shared extension PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); + // Similar to _Py_HashPointer(), but don't replace -1 with -2 extern Py_hash_t _Py_HashPointerRaw(const void*); -// _datetime shared extension uses _Py_HashBytes() + +// Export for '_datetime' shared extension PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); /* Prime multiplier used in string and various other hashes. */ From 52c6a6e48a5fa12af401810722cfcad859e9881a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 17:34:22 +0200 Subject: [PATCH 297/598] gh-108308: Remove _PyDict_GetItemStringWithError() function (#108426) Remove the internal _PyDict_GetItemStringWithError() function. It can now be replaced with the new public PyDict_ContainsString() and PyDict_GetItemStringRef() functions. getargs.c now now uses a strong reference for current_arg. find_keyword() returns a strong reference. --- Include/internal/pycore_dict.h | 1 - Objects/dictobject.c | 13 -------- Python/getargs.c | 58 ++++++++++++++++++---------------- Python/pythonrun.c | 1 - Python/sysmodule.c | 7 +++- 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 8e9c27e3b294b8..df7bc7e58f6c97 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -13,7 +13,6 @@ extern "C" { // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -extern PyObject* _PyDict_GetItemStringWithError(PyObject *, const char *); extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 10205e379a569b..d98e9a395c7eae 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1829,19 +1829,6 @@ _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference } -PyObject * -_PyDict_GetItemStringWithError(PyObject *v, const char *key) -{ - PyObject *kv, *rv; - kv = PyUnicode_FromString(key); - if (kv == NULL) { - return NULL; - } - rv = PyDict_GetItemWithError(v, kv); - Py_DECREF(kv); - return rv; // borrowed reference -} - /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * diff --git a/Python/getargs.c b/Python/getargs.c index 916e46578a454b..809b2d4ef17d3d 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1491,7 +1491,6 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, int i, pos, len; int skip = 0; Py_ssize_t nargs, nkwargs; - PyObject *current_arg; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; @@ -1621,17 +1620,17 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, return cleanreturn(0, &freelist); } if (!skip) { + PyObject *current_arg; if (i < nargs) { - current_arg = PyTuple_GET_ITEM(args, i); + current_arg = Py_NewRef(PyTuple_GET_ITEM(args, i)); } else if (nkwargs && i >= pos) { - current_arg = _PyDict_GetItemStringWithError(kwargs, kwlist[i]); + if (PyDict_GetItemStringRef(kwargs, kwlist[i], ¤t_arg) < 0) { + return cleanreturn(0, &freelist); + } if (current_arg) { --nkwargs; } - else if (PyErr_Occurred()) { - return cleanreturn(0, &freelist); - } } else { current_arg = NULL; @@ -1640,6 +1639,7 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, if (current_arg) { msg = convertitem(current_arg, &format, p_va, flags, levels, msgbuf, sizeof(msgbuf), &freelist); + Py_DECREF(current_arg); if (msg) { seterror(i+1, msg, levels, fname, custom_msg); return cleanreturn(0, &freelist); @@ -1710,8 +1710,12 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, Py_ssize_t j; /* make sure there are no arguments given by name and position */ for (i = pos; i < nargs; i++) { - current_arg = _PyDict_GetItemStringWithError(kwargs, kwlist[i]); + PyObject *current_arg; + if (PyDict_GetItemStringRef(kwargs, kwlist[i], ¤t_arg) < 0) { + return cleanreturn(0, &freelist); + } if (current_arg) { + Py_DECREF(current_arg); /* arg present in tuple and in dict */ PyErr_Format(PyExc_TypeError, "argument for %.200s%s given by name ('%s') " @@ -1721,9 +1725,6 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, kwlist[i], i+1); return cleanreturn(0, &freelist); } - else if (PyErr_Occurred()) { - return cleanreturn(0, &freelist); - } } /* make sure there are no extraneous keyword arguments */ j = 0; @@ -1985,7 +1986,7 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) /* kwname == key will normally find a match in since keyword keys should be interned strings; if not retry below in a new loop. */ if (kwname == key) { - return kwstack[i]; + return Py_NewRef(kwstack[i]); } } @@ -1993,7 +1994,7 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); assert(PyUnicode_Check(kwname)); if (_PyUnicode_EQ(kwname, key)) { - return kwstack[i]; + return Py_NewRef(kwstack[i]); } } return NULL; @@ -2013,7 +2014,6 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, PyObject *keyword; int i, pos, len; Py_ssize_t nkwargs; - PyObject *current_arg; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; PyObject *const *kwstack = NULL; @@ -2108,14 +2108,14 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, } assert(!IS_END_OF_FORMAT(*format)); + PyObject *current_arg; if (i < nargs) { - current_arg = args[i]; + current_arg = Py_NewRef(args[i]); } else if (nkwargs && i >= pos) { keyword = PyTuple_GET_ITEM(kwtuple, i - pos); if (kwargs != NULL) { - current_arg = PyDict_GetItemWithError(kwargs, keyword); - if (!current_arg && PyErr_Occurred()) { + if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { return cleanreturn(0, &freelist); } } @@ -2133,6 +2133,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, if (current_arg) { msg = convertitem(current_arg, &format, p_va, flags, levels, msgbuf, sizeof(msgbuf), &freelist); + Py_DECREF(current_arg); if (msg) { seterror(i+1, msg, levels, parser->fname, parser->custom_msg); return cleanreturn(0, &freelist); @@ -2183,10 +2184,10 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, if (nkwargs > 0) { /* make sure there are no arguments given by name and position */ for (i = pos; i < nargs; i++) { + PyObject *current_arg; keyword = PyTuple_GET_ITEM(kwtuple, i - pos); if (kwargs != NULL) { - current_arg = PyDict_GetItemWithError(kwargs, keyword); - if (!current_arg && PyErr_Occurred()) { + if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { return cleanreturn(0, &freelist); } } @@ -2194,6 +2195,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, current_arg = find_keyword(kwnames, kwstack, keyword); } if (current_arg) { + Py_DECREF(current_arg); /* arg present in tuple and in dict */ PyErr_Format(PyExc_TypeError, "argument for %.200s%s given by name ('%U') " @@ -2248,7 +2250,6 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, int i, posonly, minposonly, maxargs; int reqlimit = minkw ? maxpos + minkw : minpos; Py_ssize_t nkwargs; - PyObject *current_arg; PyObject * const *kwstack = NULL; assert(kwargs == NULL || PyDict_Check(kwargs)); @@ -2343,11 +2344,11 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, /* copy keyword args using kwtuple to drive process */ for (i = Py_MAX((int)nargs, posonly); i < maxargs; i++) { + PyObject *current_arg; if (nkwargs) { keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); if (kwargs != NULL) { - current_arg = PyDict_GetItemWithError(kwargs, keyword); - if (!current_arg && PyErr_Occurred()) { + if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { return NULL; } } @@ -2365,6 +2366,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, buf[i] = current_arg; if (current_arg) { + Py_DECREF(current_arg); --nkwargs; } else if (i < minpos || (maxpos <= i && i < reqlimit)) { @@ -2382,10 +2384,10 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, if (nkwargs > 0) { /* make sure there are no arguments given by name and position */ for (i = posonly; i < nargs; i++) { + PyObject *current_arg; keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); if (kwargs != NULL) { - current_arg = PyDict_GetItemWithError(kwargs, keyword); - if (!current_arg && PyErr_Occurred()) { + if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { return NULL; } } @@ -2393,6 +2395,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, current_arg = find_keyword(kwnames, kwstack, keyword); } if (current_arg) { + Py_DECREF(current_arg); /* arg present in tuple and in dict */ PyErr_Format(PyExc_TypeError, "argument for %.200s%s given by name ('%U') " @@ -2424,7 +2427,6 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, int i, posonly, minposonly, maxargs; int reqlimit = minkw ? maxpos + minkw : minpos; Py_ssize_t nkwargs; - PyObject *current_arg; PyObject * const *kwstack = NULL; assert(kwargs == NULL || PyDict_Check(kwargs)); @@ -2497,13 +2499,12 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, } /* copy keyword args using kwtuple to drive process */ - for (i = Py_MAX((int)nargs, posonly) - - Py_SAFE_DOWNCAST(varargssize, Py_ssize_t, int); i < maxargs; i++) { + for (i = Py_MAX((int)nargs, posonly) - Py_SAFE_DOWNCAST(varargssize, Py_ssize_t, int); i < maxargs; i++) { + PyObject *current_arg; if (nkwargs) { keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); if (kwargs != NULL) { - current_arg = PyDict_GetItemWithError(kwargs, keyword); - if (!current_arg && PyErr_Occurred()) { + if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { goto exit; } } @@ -2536,6 +2537,7 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, } if (current_arg) { + Py_DECREF(current_arg); --nkwargs; } else if (i < minpos || (maxpos <= i && i < reqlimit)) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 0e118b03d6a111..a2b50c021ce9e4 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -15,7 +15,6 @@ #include "pycore_ast.h" // PyAST_mod2obj #include "pycore_ceval.h" // _Py_EnterRecursiveCall #include "pycore_compile.h" // _PyAST_Compile() -#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_object.h" // _PyDebug_PrintTotalRefs() #include "pycore_parser.h" // _PyParser_ASTFromString() diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 54533603f1a7f8..94d0f01f7477a7 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -87,7 +87,12 @@ _PySys_GetObject(PyInterpreterState *interp, const char *name) if (sysdict == NULL) { return NULL; } - return _PyDict_GetItemStringWithError(sysdict, name); + PyObject *value; + if (PyDict_GetItemStringRef(sysdict, name, &value) != 1) { + return NULL; + } + Py_DECREF(value); // return a borrowed reference + return value; } PyObject * From f1ae706ca53d43890cf09ccf052d783e97ba4b28 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 17:40:56 +0200 Subject: [PATCH 298/598] gh-107211: No longer export internal functions (7) (#108425) No longer export _PyUnicode_FromId() internal C API function. Change comment style to "// comment" and add comment explaining why other functions have to be exported. Update Tools/build/generate_token.py to update Include/internal/pycore_token.h comments. --- Include/internal/pycore_pylifecycle.h | 4 ++ Include/internal/pycore_pystate.h | 3 + Include/internal/pycore_runtime.h | 3 + Include/internal/pycore_setobject.h | 6 +- Include/internal/pycore_sysmodule.h | 4 +- Include/internal/pycore_token.h | 4 +- Include/internal/pycore_typeobject.h | 4 +- Include/internal/pycore_unicodeobject.h | 79 ++++++++++++++----------- Modules/_testinternalcapi.c | 4 +- Tools/build/generate_token.py | 4 +- 10 files changed, 69 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 56abd57d2bd5cf..ec003a1dad2595 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -98,16 +98,20 @@ extern int _Py_FdIsInteractive(FILE *fp, PyObject *filename); extern const char* _Py_gitidentifier(void); extern const char* _Py_gitversion(void); +// Export for '_asyncio' shared extension PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp); /* Random */ extern int _PyOS_URandom(void *buffer, Py_ssize_t size); + // Export for '_random' shared extension PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); /* Legacy locale support */ extern int _Py_CoerceLegacyLocale(int warn); extern int _Py_LegacyLocaleDetected(int warn); + +// Export for 'readline' shared extension PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); #ifdef __cplusplus diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index f6ca9102b139e6..2e601cc6c4a464 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -66,6 +66,9 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) #if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) extern _Py_thread_local PyThreadState *_Py_tss_tstate; #endif + +// Export for most shared extensions, used via _PyThreadState_GET() static +// inline function. PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); /* Get the current Python thread state. diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 4a460803c69bde..2ce46f3201d8af 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -272,6 +272,9 @@ typedef struct pyruntimestate { /* other API */ +// Export _PyRuntime for shared extensions which use it in static inline +// functions for best performance, like _Py_IsMainThread() or _Py_ID(). +// It's also made accessible for debuggers and profilers. PyAPI_DATA(_PyRuntimeState) _PyRuntime; extern PyStatus _PyRuntimeState_Init(_PyRuntimeState *runtime); diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 1b63479e774412..34a00e6d45fe69 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -8,17 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// Export for 'pickle' shared extension +// Export for '_pickle' shared extension PyAPI_FUNC(int) _PySet_NextEntry( PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); -// Export for 'pickle' shared extension +// Export for '_pickle' shared extension PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); -// Export _PySet_Dummy for the gdb plugin's benefit +// Export for the gdb plugin's (python-gdb.py) benefit PyAPI_DATA(PyObject *) _PySet_Dummy; #ifdef __cplusplus diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 89a2f7628645b9..aec9c2060fd0c0 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -14,8 +14,8 @@ extern int _PySys_Audit( const char *argFormat, ...); -/* We want minimal exposure of this function, so use extern rather than - PyAPI_FUNC() to not export the symbol. */ +// _PySys_ClearAuditHooks() must not be exported: use extern rather than +// PyAPI_FUNC(). We want minimal exposure of this function. extern void _PySys_ClearAuditHooks(PyThreadState *tstate); extern int _PySys_SetAttr(PyObject *, PyObject *); diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index 9c65cd802d597c..571cd6249f2812 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -1,4 +1,4 @@ -/* Auto-generated by Tools/build/generate_token.py */ +// Auto-generated by Tools/build/generate_token.py /* Token types */ #ifndef Py_INTERNAL_TOKEN_H @@ -94,7 +94,7 @@ extern "C" { (x) == FSTRING_MIDDLE) -// Symbols exported for test_peg_generator +// Export these 4 symbols for 'test_peg_generator' PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) _PyToken_OneChar(int); PyAPI_FUNC(int) _PyToken_TwoChars(int, int); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 4d6a0189c4afee..27c6c8731cb3f9 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -114,8 +114,10 @@ extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTyp extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); -// Export for 'math' shared extension via _PyType_IsReady() function +// Export for 'math' shared extension, used via _PyType_IsReady() static inline +// function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); + extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 8ec80ddb83d10f..08c3dfec93bf05 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -24,6 +24,7 @@ extern int _PyUnicode_IsCased(Py_UCS4 ch); /* --- Unicode API -------------------------------------------------------- */ +// Export for '_json' shared extension PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); @@ -31,10 +32,10 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( extern void _PyUnicode_ExactDealloc(PyObject *op); extern Py_ssize_t _PyUnicode_InternedSize(void); -/* Get a copy of a Unicode string. */ +// Get a copy of a Unicode string. +// Export for '_datetime' shared extension. PyAPI_FUNC(PyObject*) _PyUnicode_Copy( - PyObject *unicode - ); + PyObject *unicode); /* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash if parameters are invalid (e.g. if length is longer than the string). */ @@ -93,11 +94,13 @@ typedef struct { unsigned char readonly; } _PyUnicodeWriter ; -/* Initialize a Unicode writer. - * - * By default, the minimum buffer size is 0 character and overallocation is - * disabled. Set min_length, min_char and overallocate attributes to control - * the allocation of the buffer. */ +// Initialize a Unicode writer. +// +// By default, the minimum buffer size is 0 character and overallocation is +// disabled. Set min_length, min_char and overallocate attributes to control +// the allocation of the buffer. +// +// Export the _PyUnicodeWriter API for '_multibytecodec' shared extension. PyAPI_FUNC(void) _PyUnicodeWriter_Init(_PyUnicodeWriter *writer); @@ -204,12 +207,14 @@ extern PyObject* _PyUnicode_EncodeUTF7( /* --- UTF-8 Codecs ------------------------------------------------------- */ +// Export for '_tkinter' shared extension. PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( PyObject *unicode, const char *errors); /* --- UTF-32 Codecs ------------------------------------------------------ */ +// Export for '_tkinter' shared extension PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( PyObject *object, /* Unicode object */ const char *errors, /* error handling */ @@ -217,20 +222,21 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( /* --- UTF-16 Codecs ------------------------------------------------------ */ -/* Returns a Python string object holding the UTF-16 encoded value of - the Unicode data. - - If byteorder is not 0, output is written according to the following - byte order: - - byteorder == -1: little endian - byteorder == 0: native byte order (writes a BOM mark) - byteorder == 1: big endian - - If byteorder is 0, the output string will always start with the - Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is - prepended. -*/ +// Returns a Python string object holding the UTF-16 encoded value of +// the Unicode data. +// +// If byteorder is not 0, output is written according to the following +// byte order: +// +// byteorder == -1: little endian +// byteorder == 0: native byte order (writes a BOM mark) +// byteorder == 1: big endian +// +// If byteorder is 0, the output string will always start with the +// Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is +// prepended. +// +// Export for '_tkinter' shared extension PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( PyObject* unicode, /* Unicode object */ const char *errors, /* error handling */ @@ -297,13 +303,14 @@ extern PyObject* _PyUnicode_EncodeCharmap( /* --- Decimal Encoder ---------------------------------------------------- */ -/* Coverts a Unicode object holding a decimal value to an ASCII string - for using in int, float and complex parsers. - Transforms code points that have decimal digit property to the - corresponding ASCII digit code points. Transforms spaces to ASCII. - Transforms code points starting from the first non-ASCII code point that - is neither a decimal digit nor a space to the end into '?'. */ - +// Coverts a Unicode object holding a decimal value to an ASCII string +// for using in int, float and complex parsers. +// Transforms code points that have decimal digit property to the +// corresponding ASCII digit code points. Transforms spaces to ASCII. +// Transforms code points starting from the first non-ASCII code point that +// is neither a decimal digit nor a space to the end into '?'. +// +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( PyObject *unicode); /* Unicode object */ @@ -323,9 +330,10 @@ extern int _PyUnicode_EqualToASCIIId( _Py_Identifier *right /* Right identifier */ ); -/* Test whether a unicode is equal to ASCII string. Return 1 if true, - 0 otherwise. The right argument must be ASCII-encoded string. - Any error occurs inside will be cleared before return. */ +// Test whether a unicode is equal to ASCII string. Return 1 if true, +// 0 otherwise. The right argument must be ASCII-encoded string. +// Any error occurs inside will be cleared before return. +// Export for '_ctypes' shared extension PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString( PyObject *left, const char *right /* ASCII-encoded string */ @@ -357,14 +365,17 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); -/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/ +// Return an interned Unicode object for an Identifier; may fail if there is no +// memory. +// Export for '_testembed' program. PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); /* Fast equality check when the inputs are known to be exact unicode types and where the hash values are equal (i.e. a very probable match) */ extern int _PyUnicode_EQ(PyObject *, PyObject *); -/* Equality check. */ +// Equality check. +// Export for '_pickle' shared extension. PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); extern int _PyUnicode_WideCharString_Converter(PyObject *, void *); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 5571ae463373e0..7b98885189f5c0 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,9 +14,9 @@ #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc -#include "pycore_ceval.h" // _PyEval_AddPendingCall +#include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_dict.h" // _PyDictOrValues_GetValues() -#include "pycore_fileutils.h" // _Py_normpath +#include "pycore_fileutils.h" // _Py_normpath() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 3bd307c1733867..16c38841e44a4d 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -50,7 +50,7 @@ def update_file(file, content): token_h_template = f"""\ -/* {AUTO_GENERATED_BY_SCRIPT} */ +// {AUTO_GENERATED_BY_SCRIPT} """ token_h_template += """\ @@ -84,7 +84,7 @@ def update_file(file, content): (x) == FSTRING_MIDDLE) -// Symbols exported for test_peg_generator +// Export these 4 symbols for 'test_peg_generator' PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) _PyToken_OneChar(int); PyAPI_FUNC(int) _PyToken_TwoChars(int, int); From b7808820b12b2df30cb21d8e8a5634e3ff97e376 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 18:06:53 +0200 Subject: [PATCH 299/598] gh-107211: No longer export internal functions (5) (#108423) No longer export _PyCompile_AstOptimize() internal C API function. Change comment style to "// comment" and add comment explaining why other functions have to be exported. --- Include/internal/pycore_bytesobject.h | 41 ++++++++++++++----------- Include/internal/pycore_call.h | 8 ++--- Include/internal/pycore_ceval.h | 5 ++- Include/internal/pycore_compile.h | 5 ++- Include/internal/pycore_complexobject.h | 4 +-- Include/internal/pycore_fileutils.h | 20 ++++++++++-- Include/internal/pycore_genobject.h | 1 + Include/internal/pycore_hashtable.h | 9 +++--- Include/internal/pycore_import.h | 9 ++++-- Include/internal/pycore_initconfig.h | 10 +++--- 10 files changed, 73 insertions(+), 39 deletions(-) diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 980065a65f399e..94d421a9eb742a 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -28,34 +28,37 @@ PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, extern PyObject* _PyBytes_Join(PyObject *sep, PyObject *x); -/* Substring Search. - - Returns the index of the first occurrence of - a substring ("needle") in a larger text ("haystack"). - If the needle is not found, return -1. - If the needle is found, add offset to the index. -*/ - +// Substring Search. +// +// Returns the index of the first occurrence of +// a substring ("needle") in a larger text ("haystack"). +// If the needle is not found, return -1. +// If the needle is found, add offset to the index. +// +// Export for 'mmap' shared extension. PyAPI_FUNC(Py_ssize_t) _PyBytes_Find(const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset); -/* Same as above, but search right-to-left */ +// Same as above, but search right-to-left. +// Export for 'mmap' shared extension. PyAPI_FUNC(Py_ssize_t) _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset); -/** Helper function to implement the repeat and inplace repeat methods on a buffer - * - * len_dest is assumed to be an integer multiple of len_src. - * If src equals dest, then assume the operation is inplace. - * - * This method repeately doubles the number of bytes copied to reduce - * the number of invocations of memcpy. - */ +// Helper function to implement the repeat and inplace repeat methods on a +// buffer. +// +// len_dest is assumed to be an integer multiple of len_src. +// If src equals dest, then assume the operation is inplace. +// +// This method repeately doubles the number of bytes copied to reduce +// the number of invocations of memcpy. +// +// Export for 'array' shared extension. PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); @@ -91,7 +94,9 @@ typedef struct { /* Initialize a bytes writer By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. */ + to control the allocation of the buffer. + + Export _PyBytesWriter API for '_pickle' shared extension. */ PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); /* Get the buffer content and reset the writer. diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index c0d61785802f64..db719234e05bf3 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -22,8 +22,8 @@ extern "C" { #define _PY_FASTCALL_SMALL_STACK 5 -// Export for 'math' shared extension, function used -// via inlined _PyObject_VectorcallTstate() function. +// Export for 'math' shared extension, used via _PyObject_VectorcallTstate() +// static inline function. PyAPI_FUNC(PyObject*) _Py_CheckFunctionResult( PyThreadState *tstate, PyObject *callable, @@ -120,8 +120,8 @@ _PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg // Call callable using tp_call. Arguments are like PyObject_Vectorcall(), // except that nargs is plainly the number of arguments without flags. // -// Export for 'math' shared extension, function used -// via inlined _PyObject_VectorcallTstate() function. +// Export for 'math' shared extension, used via _PyObject_VectorcallTstate() +// static inline function. PyAPI_FUNC(PyObject*) _PyObject_MakeTpCall( PyThreadState *tstate, PyObject *callable, diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index f32ed3b7e30b7e..db3dac48662326 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -23,12 +23,14 @@ extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); extern void _PyEval_SignalReceived(PyInterpreterState *interp); + // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyEval_AddPendingCall( PyInterpreterState *interp, int (*func)(void *), void *arg, int mainthreadonly); + extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); @@ -122,7 +124,8 @@ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { } #endif -// Export for _Py_EnterRecursiveCall() +// Export for '_json' shared extension, used via _Py_EnterRecursiveCall() +// static inline function. PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 0167e590e63fb1..a5a7146f5ee917 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -20,7 +20,7 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( struct _arena *arena); /* AST optimizations */ -PyAPI_FUNC(int) _PyCompile_AstOptimize( +extern int _PyCompile_AstOptimize( struct _mod *mod, PyObject *filename, PyCompilerFlags *flags, @@ -107,6 +107,7 @@ int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyCompile_CleanDoc(PyObject *doc); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, @@ -114,11 +115,13 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( int optimize, int compile_mode); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, PyObject *consts, int nlocals); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyCodeObject*) _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, PyObject *instructions); diff --git a/Include/internal/pycore_complexobject.h b/Include/internal/pycore_complexobject.h index 7843c0a008ebb6..a6fee9d23f3a9f 100644 --- a/Include/internal/pycore_complexobject.h +++ b/Include/internal/pycore_complexobject.h @@ -10,8 +10,8 @@ extern "C" { #include "pycore_unicodeobject.h" // _PyUnicodeWriter -/* Operations on complex numbers from complexmodule.c */ - +// Operations on complex numbers. +// Export functions for 'cmath' shared extension. PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 96e3d1c7abcdde..90c0878a3ee7d6 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -35,8 +35,10 @@ typedef enum { _Py_ERROR_OTHER } _Py_error_handler; +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _Py_DecodeLocaleEx( const char *arg, wchar_t **wstr, @@ -45,6 +47,7 @@ PyAPI_FUNC(int) _Py_DecodeLocaleEx( int current_locale, _Py_error_handler errors); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _Py_EncodeLocaleEx( const wchar_t *text, char **str, @@ -98,23 +101,27 @@ struct _Py_stat_struct { # define _Py_stat_struct stat #endif +// Export for 'mmap' shared extension PyAPI_FUNC(int) _Py_fstat( int fd, struct _Py_stat_struct *status); +// Export for 'mmap' shared extension PyAPI_FUNC(int) _Py_fstat_noraise( int fd, struct _Py_stat_struct *status); +// Export for '_tkinter' shared extension PyAPI_FUNC(int) _Py_stat( PyObject *path, struct stat *status); -// Export for 'select' shared extension (Solaris newDevPollObject() uses it) +// Export for 'select' shared extension (Solaris newDevPollObject()) PyAPI_FUNC(int) _Py_open( const char *pathname, int flags); +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(int) _Py_open_noraise( const char *pathname, int flags); @@ -128,12 +135,13 @@ extern Py_ssize_t _Py_read( void *buf, size_t count); -// Export for 'select' shared extension (Solaris devpoll_flush() uses it) +// Export for 'select' shared extension (Solaris devpoll_flush()) PyAPI_FUNC(Py_ssize_t) _Py_write( int fd, const void *buf, size_t count); +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( int fd, const void *buf, @@ -165,12 +173,15 @@ extern wchar_t* _Py_wgetcwd( extern int _Py_get_inheritable(int fd); +// Export for '_socket' shared extension PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, int *atomic_flag_works); +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, int *atomic_flag_works); +// Export for '_socket' shared extension PyAPI_FUNC(int) _Py_dup(int fd); extern int _Py_get_blocking(int fd); @@ -180,6 +191,7 @@ extern int _Py_set_blocking(int fd, int blocking); #ifdef MS_WINDOWS extern void* _Py_get_osfhandle_noraise(int fd); +// Export for '_testconsole' shared extension PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); extern int _Py_open_osfhandle_noraise(void *handle, int flags); @@ -234,6 +246,7 @@ extern int _Py_GetLocaleconvNumeric( PyObject **decimal_point, PyObject **thousands_sep); +// Export for '_posixsubprocess' (on macOS) PyAPI_FUNC(void) _Py_closerange(int first, int last); extern wchar_t* _Py_GetLocaleEncoding(void); @@ -262,7 +275,10 @@ extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); + +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); + extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); // The Windows Games API family does not provide these functions diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 96a6ec43d7a08a..cf58a2750a31f9 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -13,6 +13,7 @@ extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); + // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); diff --git a/Include/internal/pycore_hashtable.h b/Include/internal/pycore_hashtable.h index f57978a8d614fb..369d49c42bbfcc 100644 --- a/Include/internal/pycore_hashtable.h +++ b/Include/internal/pycore_hashtable.h @@ -70,6 +70,11 @@ struct _Py_hashtable_t { _Py_hashtable_allocator_t alloc; }; +// Export _Py_hashtable functions for '_testinternalcapi' shared extension +PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + _Py_hashtable_hash_func hash_func, + _Py_hashtable_compare_func compare_func); + /* Hash a pointer (void*) */ PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); @@ -78,10 +83,6 @@ PyAPI_FUNC(int) _Py_hashtable_compare_direct( const void *key1, const void *key2); -PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( - _Py_hashtable_hash_func hash_func, - _Py_hashtable_compare_func compare_func); - PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index e23f9931cea734..117e46bb86285d 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -14,7 +14,9 @@ extern "C" { extern int _PyImport_IsInitialized(PyInterpreterState *); +// Export for 'pyexpat' shared extension PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); + extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); @@ -28,7 +30,10 @@ extern int _PyImport_FixupBuiltin( extern int _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *, PyObject *); +// Export for many shared extensions, like '_json' PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttr(PyObject *, PyObject *); + +// Export for many shared extensions, like '_datetime' PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttrString(const char *, const char *); @@ -187,11 +192,9 @@ struct _module_alias { const char *orig; /* ASCII encoded string */ }; -// Export for test_ctypes +// Export these 3 symbols for test_ctypes PyAPI_DATA(const struct _frozen*) _PyImport_FrozenBootstrap; -// Export for test_ctypes PyAPI_DATA(const struct _frozen*) _PyImport_FrozenStdlib; -// Export for test_ctypes PyAPI_DATA(const struct _frozen*) _PyImport_FrozenTest; extern const struct _module_alias * _PyImport_FrozenAliases; diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 0945bb0936f039..c9645c7dc66573 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -124,6 +124,7 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline, // Export for '_testembed' program PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); + extern void _PyPreConfig_InitFromConfig( PyPreConfig *preconfig, const PyConfig *config); @@ -149,6 +150,7 @@ typedef enum { // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); + extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); @@ -163,16 +165,16 @@ extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); -PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); -PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); extern void _Py_DumpPathConfig(PyThreadState *tstate); -PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(void); - /* --- Function used for testing ---------------------------------- */ +// Export these functions for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); +PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); +PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(void); PyAPI_FUNC(PyObject*) _Py_GetConfigsAsDict(void); #ifdef __cplusplus From 7f316763402a7d5556deecc3acd06cb719e189b3 Mon Sep 17 00:00:00 2001 From: A <5249513+Dumeng@users.noreply.github.com> Date: Fri, 25 Aug 2023 00:27:54 +0800 Subject: [PATCH 300/598] Fix a code snippet typo in asyncio docs (#108427) --- Doc/library/asyncio-task.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 1c419728c9a482..39c1e939d91652 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -592,7 +592,7 @@ Shielding From Cancellation is equivalent to:: - res = await something() + res = await shield(something()) *except* that if the coroutine containing it is cancelled, the Task running in ``something()`` is not cancelled. From the point From c55e73112c7b0574d05a522015d0c927de266525 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 18:53:50 +0200 Subject: [PATCH 301/598] gh-106320: Remove private PyLong C API functions (#108429) Remove private PyLong C API functions: * _PyLong_AsByteArray() * _PyLong_DivmodNear() * _PyLong_Format() * _PyLong_Frexp() * _PyLong_FromByteArray() * _PyLong_FromBytes() * _PyLong_GCD() * _PyLong_Lshift() * _PyLong_Rshift() Move these functions to the internal C API. No longer export _PyLong_FromBytes() function. --- Include/cpython/longobject.h | 70 +------------------------ Include/internal/pycore_long.h | 84 ++++++++++++++++++++++++++++-- Modules/_io/_iomodule.c | 5 +- Modules/_pickle.c | 1 + Modules/_randommodule.c | 2 +- Modules/_sqlite/util.c | 5 ++ Modules/_struct.c | 1 + Modules/arraymodule.c | 3 +- Modules/cjkcodecs/multibytecodec.c | 1 + Python/hamt.c | 5 +- 10 files changed, 99 insertions(+), 78 deletions(-) diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 90cc0f267ae833..c581f51cbcdf38 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -10,16 +10,7 @@ PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); -/* _PyLong_Frexp returns a double x and an exponent e such that the - true value is approximately equal to x * 2**e. e is >= 0. x is - 0.0 if and only if the input is 0 (in which case, e and x are both - zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is - possible if the number of bits doesn't fit into a Py_ssize_t, sets - OverflowError and returns -1.0 for x, 0 for e. */ -PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); - -PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); +PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); /* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. v must not be NULL, and must be a normalized long. @@ -36,65 +27,6 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); */ PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); -/* _PyLong_DivmodNear. Given integers a and b, compute the nearest - integer q to the exact quotient a / b, rounding to the nearest even integer - in the case of a tie. Return (q, r), where r = a - q*b. The remainder r - will satisfy abs(r) <= abs(b)/2, with equality possible only if q is - even. -*/ -PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); - -/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in - base 256, and return a Python int with the same numeric value. - If n is 0, the integer is 0. Else: - If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; - else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the - LSB. - If is_signed is 0/false, view the bytes as a non-negative integer. - If is_signed is 1/true, view the bytes as a 2's-complement integer, - non-negative if bit 0x80 of the MSB is clear, negative if set. - Error returns: - + Return NULL with the appropriate exception set if there's not - enough memory to create the Python int. -*/ -PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( - const unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long - v to a base-256 integer, stored in array bytes. Normally return 0, - return -1 on error. - If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at - bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and - the LSB at bytes[n-1]. - If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes - are filled and there's nothing special about bit 0x80 of the MSB. - If is_signed is 1/true, bytes is filled with the 2's-complement - representation of v's value. Bit 0x80 of the MSB is the sign bit. - Error returns (-1): - + is_signed is 0 and v < 0. TypeError is set in this case, and bytes - isn't altered. - + n isn't big enough to hold the full mathematical value of v. For - example, if is_signed is 0 and there are more digits in the v than - fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of - being large enough to hold a sign bit. OverflowError is set in this - case, but bytes holds the least-significant n bytes of the true value. -*/ -PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, - unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_Format: Convert the long to a string object with given base, - appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); - -/* For use by the gcd function in mathmodule.c */ -PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); - -PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); -PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); - - PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 4eb70f7e3709dc..a76494808bffdd 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -79,9 +79,87 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]; } -extern PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); -extern PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); -extern PyObject *_PyLong_Subtract(PyLongObject *left, PyLongObject *right); +// _PyLong_Frexp returns a double x and an exponent e such that the +// true value is approximately equal to x * 2**e. e is >= 0. x is +// 0.0 if and only if the input is 0 (in which case, e and x are both +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is +// possible if the number of bits doesn't fit into a Py_ssize_t, sets +// OverflowError and returns -1.0 for x, 0 for e. +// +// Export for 'math' shared extension +PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); + +extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int); + +// _PyLong_DivmodNear. Given integers a and b, compute the nearest +// integer q to the exact quotient a / b, rounding to the nearest even integer +// in the case of a tie. Return (q, r), where r = a - q*b. The remainder r +// will satisfy abs(r) <= abs(b)/2, with equality possible only if q is +// even. +// +// Export for '_datetime' shared extension. +PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); + +// _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in +// base 256, and return a Python int with the same numeric value. +// If n is 0, the integer is 0. Else: +// If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; +// else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the +// LSB. +// If is_signed is 0/false, view the bytes as a non-negative integer. +// If is_signed is 1/true, view the bytes as a 2's-complement integer, +// non-negative if bit 0x80 of the MSB is clear, negative if set. +// Error returns: +// + Return NULL with the appropriate exception set if there's not +// enough memory to create the Python int. +// +// Export for '_multibytecodec' shared extension. +PyAPI_DATA(PyObject*) _PyLong_FromByteArray( + const unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +// _PyLong_AsByteArray: Convert the least-significant 8*n bits of long +// v to a base-256 integer, stored in array bytes. Normally return 0, +// return -1 on error. +// If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at +// bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and +// the LSB at bytes[n-1]. +// If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes +// are filled and there's nothing special about bit 0x80 of the MSB. +// If is_signed is 1/true, bytes is filled with the 2's-complement +// representation of v's value. Bit 0x80 of the MSB is the sign bit. +// Error returns (-1): +// + is_signed is 0 and v < 0. TypeError is set in this case, and bytes +// isn't altered. +// + n isn't big enough to hold the full mathematical value of v. For +// example, if is_signed is 0 and there are more digits in the v than +// fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of +// being large enough to hold a sign bit. OverflowError is set in this +// case, but bytes holds the least-significant n bytes of the true value. +// +// Export for '_struct' shared extension. +PyAPI_DATA(int) _PyLong_AsByteArray(PyLongObject* v, + unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +// _PyLong_Format: Convert the long to a string object with given base, +// appending a base prefix of 0[box] if base is 2, 8 or 16. +// Export for '_tkinter' shared extension. +PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); + +// For use by the math.gcd() function. +// Export for 'math' shared extension. +PyAPI_DATA(PyObject*) _PyLong_GCD(PyObject *, PyObject *); + +// Export for 'math' shared extension +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); + +// Export for 'math' shared extension +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); + +extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right); +extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right); +extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 1a7920eebe0b61..f56654227f1020 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -8,9 +8,10 @@ */ #include "Python.h" -#include "_iomodule.h" -#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_pystate.h" // _PyInterpreterState_GET() + +#include "_iomodule.h" #ifdef HAVE_SYS_TYPES_H #include diff --git a/Modules/_pickle.c b/Modules/_pickle.c index c2b04cc513a664..4f26ffe27d75f1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -11,6 +11,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyNone_Type #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 7daa1f9327966f..18811d03adb451 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -71,9 +71,9 @@ #endif #include "Python.h" +#include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() -#include "pycore_runtime.h" #ifdef HAVE_PROCESS_H # include // getpid() #endif diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 2b3bbfefa3cf5f..833a666301d8ff 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -21,7 +21,12 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "module.h" +#include "pycore_long.h" // _PyLong_AsByteArray() #include "connection.h" // Returns non-NULL if a new exception should be raised diff --git a/Modules/_struct.c b/Modules/_struct.c index 425715ad030d4d..606ae5e4e19964 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a40a5b75b63d4f..5c1e6cb9cffbc3 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -8,9 +8,10 @@ #endif #include "Python.h" +#include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() +#include "pycore_long.h" // _PyLong_FromByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_bytesobject.h" // _PyBytes_Repeat #include // offsetof() #include diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5d3c16a98423ba..7c1da9004549a6 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -9,6 +9,7 @@ #endif #include "Python.h" +#include "pycore_long.h" // _PyLong_FromByteArray() #include "multibytecodec.h" #include "clinic/multibytecodec.c.h" diff --git a/Python/hamt.c b/Python/hamt.c index 24265edc2c3fd4..a8fbb00b807934 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -1,9 +1,10 @@ #include "Python.h" - -#include "pycore_bitutils.h" // _Py_popcount32 +#include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_hamt.h" #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_long.h" // _PyLong_Format() #include "pycore_object.h" // _PyObject_GC_TRACK() + #include // offsetof() /* From 773b803c02b25b58af0a64321fbe1c6bf45cba2b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 19:07:54 +0200 Subject: [PATCH 302/598] gh-106320: Remove private _PyManagedBuffer_Type (#108431) Remove private _PyManagedBuffer_Type variable. Move it to the internal C API and no longer export it. --- Include/cpython/memoryobject.h | 2 -- Include/internal/pycore_memoryobject.h | 2 ++ Objects/memoryobject.c | 1 + Objects/object.c | 4 +++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index 3837fa8c6ab5aa..961161b70f2058 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -2,8 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_DATA(PyTypeObject) _PyManagedBuffer_Type; - /* The structs are declared here so that macros can work, but they shouldn't be considered public. Don't access their fields directly, use the macros and functions instead! */ diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h index fe19e3f9611a16..62e204fcbf6533 100644 --- a/Include/internal/pycore_memoryobject.h +++ b/Include/internal/pycore_memoryobject.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern PyTypeObject _PyManagedBuffer_Type; + PyObject * _PyMemoryView_FromBufferProc(PyObject *v, int flags, getbufferproc bufferproc); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b0168044d9f85a..bcdd2ff0ceafe6 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -12,6 +12,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_strhex.h" // _Py_strhex_with_sep() #include // offsetof() diff --git a/Objects/object.c b/Objects/object.c index 868623a9f7bffc..02e2611020983d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -9,15 +9,17 @@ #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type -#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_typeobject.h" // _PyBufferWrapper_Type +#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type + #include "interpreteridobject.h" // _PyInterpreterID_Type #ifdef Py_LIMITED_API From bbbe1faf7bc6860d4a628e204db944b81dfdbd73 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 19:09:49 +0200 Subject: [PATCH 303/598] gh-106320: Remove private float C API functions (#108430) 106320: Remove private float C API functions Remove private C API functions: * _Py_parse_inf_or_nan() * _Py_string_to_number_with_underscores() Move these functions to the internal C API and no longer export them. --- Include/internal/pycore_floatobject.h | 7 +++++++ Include/pystrtod.h | 9 --------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 6b9af03c25ec36..4e5474841bc25d 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -67,6 +67,13 @@ extern int _PyFloat_FormatAdvancedWriter( Py_ssize_t start, Py_ssize_t end); +extern PyObject* _Py_string_to_number_with_underscores( + const char *str, Py_ssize_t len, const char *what, PyObject *obj, void *arg, + PyObject *(*innerfunc)(const char *, Py_ssize_t, void *)); + +extern double _Py_parse_inf_or_nan(const char *p, char **endptr); + + #ifdef __cplusplus } #endif diff --git a/Include/pystrtod.h b/Include/pystrtod.h index fa056d17b6395f..e83d245eb623af 100644 --- a/Include/pystrtod.h +++ b/Include/pystrtod.h @@ -18,15 +18,6 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val, int flags, int *type); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _Py_string_to_number_with_underscores( - const char *str, Py_ssize_t len, const char *what, PyObject *obj, void *arg, - PyObject *(*innerfunc)(const char *, Py_ssize_t, void *)); - -PyAPI_FUNC(double) _Py_parse_inf_or_nan(const char *p, char **endptr); -#endif - - /* PyOS_double_to_string's "flags" parameter can be set to 0 or more of: */ #define Py_DTSF_SIGN 0x01 /* always add the sign */ #define Py_DTSF_ADD_DOT_0 0x02 /* if the result is an integer add ".0" */ From aa6f787faa4bc45006da4dc2f942fb9b82c98836 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 19:21:44 +0200 Subject: [PATCH 304/598] gh-108388: Convert test_concurrent_futures to package (#108401) Convert test_concurrent_futures to a package of sub-tests. --- Lib/test/libregrtest/runtest.py | 1 + Lib/test/test_concurrent_futures.py | 1677 ----------------- Lib/test/test_concurrent_futures/__init__.py | 16 + Lib/test/test_concurrent_futures/executor.py | 107 ++ .../test_as_completed.py | 115 ++ .../test_concurrent_futures/test_deadlock.py | 253 +++ .../test_concurrent_futures/test_future.py | 291 +++ Lib/test/test_concurrent_futures/test_init.py | 117 ++ .../test_process_pool.py | 202 ++ .../test_concurrent_futures/test_shutdown.py | 343 ++++ .../test_thread_pool.py | 98 + Lib/test/test_concurrent_futures/test_wait.py | 161 ++ Lib/test/test_concurrent_futures/util.py | 141 ++ ...-08-24-06-10-36.gh-issue-108388.YCVB0D.rst | 2 + 14 files changed, 1847 insertions(+), 1677 deletions(-) delete mode 100644 Lib/test/test_concurrent_futures.py create mode 100644 Lib/test/test_concurrent_futures/__init__.py create mode 100644 Lib/test/test_concurrent_futures/executor.py create mode 100644 Lib/test/test_concurrent_futures/test_as_completed.py create mode 100644 Lib/test/test_concurrent_futures/test_deadlock.py create mode 100644 Lib/test/test_concurrent_futures/test_future.py create mode 100644 Lib/test/test_concurrent_futures/test_init.py create mode 100644 Lib/test/test_concurrent_futures/test_process_pool.py create mode 100644 Lib/test/test_concurrent_futures/test_shutdown.py create mode 100644 Lib/test/test_concurrent_futures/test_thread_pool.py create mode 100644 Lib/test/test_concurrent_futures/test_wait.py create mode 100644 Lib/test/test_concurrent_futures/util.py create mode 100644 Misc/NEWS.d/next/Tests/2023-08-24-06-10-36.gh-issue-108388.YCVB0D.rst diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index cddb37c87aa4d4..fd49927679bdea 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -132,6 +132,7 @@ def __str__(self) -> str: SPLITTESTDIRS = { "test_asyncio", + "test_concurrent_futures", "test_multiprocessing_fork", "test_multiprocessing_forkserver", "test_multiprocessing_spawn", diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py deleted file mode 100644 index 0f0ea6190def06..00000000000000 --- a/Lib/test/test_concurrent_futures.py +++ /dev/null @@ -1,1677 +0,0 @@ -from test import support -from test.support import import_helper -from test.support import threading_helper - -# Skip tests if _multiprocessing wasn't built. -import_helper.import_module('_multiprocessing') - -from test.support import hashlib_helper -from test.support.script_helper import assert_python_ok - -import contextlib -import itertools -import logging -from logging.handlers import QueueHandler -import os -import queue -import signal -import sys -import threading -import time -import unittest -import weakref -from pickle import PicklingError - -from concurrent import futures -from concurrent.futures._base import ( - PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future, - BrokenExecutor) -from concurrent.futures.process import BrokenProcessPool, _check_system_limits - -import multiprocessing.process -import multiprocessing.util -import multiprocessing as mp - - -if support.check_sanitizer(address=True, memory=True): - # gh-90791: Skip the test because it is too slow when Python is built - # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. - raise unittest.SkipTest("test too slow on ASAN/MSAN build") - - -def create_future(state=PENDING, exception=None, result=None): - f = Future() - f._state = state - f._exception = exception - f._result = result - return f - - -PENDING_FUTURE = create_future(state=PENDING) -RUNNING_FUTURE = create_future(state=RUNNING) -CANCELLED_FUTURE = create_future(state=CANCELLED) -CANCELLED_AND_NOTIFIED_FUTURE = create_future(state=CANCELLED_AND_NOTIFIED) -EXCEPTION_FUTURE = create_future(state=FINISHED, exception=OSError()) -SUCCESSFUL_FUTURE = create_future(state=FINISHED, result=42) - -INITIALIZER_STATUS = 'uninitialized' - -def mul(x, y): - return x * y - -def capture(*args, **kwargs): - return args, kwargs - -def sleep_and_raise(t): - time.sleep(t) - raise Exception('this is an exception') - -def sleep_and_print(t, msg): - time.sleep(t) - print(msg) - sys.stdout.flush() - -def init(x): - global INITIALIZER_STATUS - INITIALIZER_STATUS = x - -def get_init_status(): - return INITIALIZER_STATUS - -def init_fail(log_queue=None): - if log_queue is not None: - logger = logging.getLogger('concurrent.futures') - logger.addHandler(QueueHandler(log_queue)) - logger.setLevel('CRITICAL') - logger.propagate = False - time.sleep(0.1) # let some futures be scheduled - raise ValueError('error in initializer') - - -class MyObject(object): - def my_method(self): - pass - - -class EventfulGCObj(): - def __init__(self, mgr): - self.event = mgr.Event() - - def __del__(self): - self.event.set() - - -def make_dummy_object(_): - return MyObject() - - -class BaseTestCase(unittest.TestCase): - def setUp(self): - self._thread_key = threading_helper.threading_setup() - - def tearDown(self): - support.reap_children() - threading_helper.threading_cleanup(*self._thread_key) - - -class ExecutorMixin: - worker_count = 5 - executor_kwargs = {} - - def setUp(self): - super().setUp() - - self.t1 = time.monotonic() - if hasattr(self, "ctx"): - self.executor = self.executor_type( - max_workers=self.worker_count, - mp_context=self.get_context(), - **self.executor_kwargs) - else: - self.executor = self.executor_type( - max_workers=self.worker_count, - **self.executor_kwargs) - - def tearDown(self): - self.executor.shutdown(wait=True) - self.executor = None - - dt = time.monotonic() - self.t1 - if support.verbose: - print("%.2fs" % dt, end=' ') - self.assertLess(dt, 300, "synchronization issue: test lasted too long") - - super().tearDown() - - def get_context(self): - return mp.get_context(self.ctx) - - -class ThreadPoolMixin(ExecutorMixin): - executor_type = futures.ThreadPoolExecutor - - -class ProcessPoolForkMixin(ExecutorMixin): - executor_type = futures.ProcessPoolExecutor - ctx = "fork" - - def get_context(self): - try: - _check_system_limits() - except NotImplementedError: - self.skipTest("ProcessPoolExecutor unavailable on this system") - if sys.platform == "win32": - self.skipTest("require unix system") - return super().get_context() - - -class ProcessPoolSpawnMixin(ExecutorMixin): - executor_type = futures.ProcessPoolExecutor - ctx = "spawn" - - def get_context(self): - try: - _check_system_limits() - except NotImplementedError: - self.skipTest("ProcessPoolExecutor unavailable on this system") - return super().get_context() - - -class ProcessPoolForkserverMixin(ExecutorMixin): - executor_type = futures.ProcessPoolExecutor - ctx = "forkserver" - - def get_context(self): - try: - _check_system_limits() - except NotImplementedError: - self.skipTest("ProcessPoolExecutor unavailable on this system") - if sys.platform == "win32": - self.skipTest("require unix system") - return super().get_context() - - -def create_executor_tests(mixin, bases=(BaseTestCase,), - executor_mixins=(ThreadPoolMixin, - ProcessPoolForkMixin, - ProcessPoolForkserverMixin, - ProcessPoolSpawnMixin)): - def strip_mixin(name): - if name.endswith(('Mixin', 'Tests')): - return name[:-5] - elif name.endswith('Test'): - return name[:-4] - else: - return name - - for exe in executor_mixins: - name = ("%s%sTest" - % (strip_mixin(exe.__name__), strip_mixin(mixin.__name__))) - cls = type(name, (mixin,) + (exe,) + bases, {}) - globals()[name] = cls - - -class InitializerMixin(ExecutorMixin): - worker_count = 2 - - def setUp(self): - global INITIALIZER_STATUS - INITIALIZER_STATUS = 'uninitialized' - self.executor_kwargs = dict(initializer=init, - initargs=('initialized',)) - super().setUp() - - def test_initializer(self): - futures = [self.executor.submit(get_init_status) - for _ in range(self.worker_count)] - - for f in futures: - self.assertEqual(f.result(), 'initialized') - - -class FailingInitializerMixin(ExecutorMixin): - worker_count = 2 - - def setUp(self): - if hasattr(self, "ctx"): - # Pass a queue to redirect the child's logging output - self.mp_context = self.get_context() - self.log_queue = self.mp_context.Queue() - self.executor_kwargs = dict(initializer=init_fail, - initargs=(self.log_queue,)) - else: - # In a thread pool, the child shares our logging setup - # (see _assert_logged()) - self.mp_context = None - self.log_queue = None - self.executor_kwargs = dict(initializer=init_fail) - super().setUp() - - def test_initializer(self): - with self._assert_logged('ValueError: error in initializer'): - try: - future = self.executor.submit(get_init_status) - except BrokenExecutor: - # Perhaps the executor is already broken - pass - else: - with self.assertRaises(BrokenExecutor): - future.result() - - # At some point, the executor should break - for _ in support.sleeping_retry(support.SHORT_TIMEOUT, - "executor not broken"): - if self.executor._broken: - break - - # ... and from this point submit() is guaranteed to fail - with self.assertRaises(BrokenExecutor): - self.executor.submit(get_init_status) - - @contextlib.contextmanager - def _assert_logged(self, msg): - if self.log_queue is not None: - yield - output = [] - try: - while True: - output.append(self.log_queue.get_nowait().getMessage()) - except queue.Empty: - pass - else: - with self.assertLogs('concurrent.futures', 'CRITICAL') as cm: - yield - output = cm.output - self.assertTrue(any(msg in line for line in output), - output) - - -create_executor_tests(InitializerMixin) -create_executor_tests(FailingInitializerMixin) - - -class ExecutorShutdownTest: - def test_run_after_shutdown(self): - self.executor.shutdown() - self.assertRaises(RuntimeError, - self.executor.submit, - pow, 2, 5) - - def test_interpreter_shutdown(self): - # Test the atexit hook for shutdown of worker threads and processes - rc, out, err = assert_python_ok('-c', """if 1: - from concurrent.futures import {executor_type} - from time import sleep - from test.test_concurrent_futures import sleep_and_print - if __name__ == "__main__": - context = '{context}' - if context == "": - t = {executor_type}(5) - else: - from multiprocessing import get_context - context = get_context(context) - t = {executor_type}(5, mp_context=context) - t.submit(sleep_and_print, 1.0, "apple") - """.format(executor_type=self.executor_type.__name__, - context=getattr(self, "ctx", ""))) - # Errors in atexit hooks don't change the process exit code, check - # stderr manually. - self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") - - def test_submit_after_interpreter_shutdown(self): - # Test the atexit hook for shutdown of worker threads and processes - rc, out, err = assert_python_ok('-c', """if 1: - import atexit - @atexit.register - def run_last(): - try: - t.submit(id, None) - except RuntimeError: - print("runtime-error") - raise - from concurrent.futures import {executor_type} - if __name__ == "__main__": - context = '{context}' - if not context: - t = {executor_type}(5) - else: - from multiprocessing import get_context - context = get_context(context) - t = {executor_type}(5, mp_context=context) - t.submit(id, 42).result() - """.format(executor_type=self.executor_type.__name__, - context=getattr(self, "ctx", ""))) - # Errors in atexit hooks don't change the process exit code, check - # stderr manually. - self.assertIn("RuntimeError: cannot schedule new futures", err.decode()) - self.assertEqual(out.strip(), b"runtime-error") - - def test_hang_issue12364(self): - fs = [self.executor.submit(time.sleep, 0.1) for _ in range(50)] - self.executor.shutdown() - for f in fs: - f.result() - - def test_cancel_futures(self): - assert self.worker_count <= 5, "test needs few workers" - fs = [self.executor.submit(time.sleep, .1) for _ in range(50)] - self.executor.shutdown(cancel_futures=True) - # We can't guarantee the exact number of cancellations, but we can - # guarantee that *some* were cancelled. With few workers, many of - # the submitted futures should have been cancelled. - cancelled = [fut for fut in fs if fut.cancelled()] - self.assertGreater(len(cancelled), 20) - - # Ensure the other futures were able to finish. - # Use "not fut.cancelled()" instead of "fut.done()" to include futures - # that may have been left in a pending state. - others = [fut for fut in fs if not fut.cancelled()] - for fut in others: - self.assertTrue(fut.done(), msg=f"{fut._state=}") - self.assertIsNone(fut.exception()) - - # Similar to the number of cancelled futures, we can't guarantee the - # exact number that completed. But, we can guarantee that at least - # one finished. - self.assertGreater(len(others), 0) - - def test_hang_gh83386(self): - """shutdown(wait=False) doesn't hang at exit with running futures. - - See https://github.com/python/cpython/issues/83386. - """ - if self.executor_type == futures.ProcessPoolExecutor: - raise unittest.SkipTest( - "Hangs, see https://github.com/python/cpython/issues/83386") - - rc, out, err = assert_python_ok('-c', """if True: - from concurrent.futures import {executor_type} - from test.test_concurrent_futures import sleep_and_print - if __name__ == "__main__": - if {context!r}: multiprocessing.set_start_method({context!r}) - t = {executor_type}(max_workers=3) - t.submit(sleep_and_print, 1.0, "apple") - t.shutdown(wait=False) - """.format(executor_type=self.executor_type.__name__, - context=getattr(self, 'ctx', None))) - self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") - - def test_hang_gh94440(self): - """shutdown(wait=True) doesn't hang when a future was submitted and - quickly canceled right before shutdown. - - See https://github.com/python/cpython/issues/94440. - """ - if not hasattr(signal, 'alarm'): - raise unittest.SkipTest( - "Tested platform does not support the alarm signal") - - def timeout(_signum, _frame): - raise RuntimeError("timed out waiting for shutdown") - - kwargs = {} - if getattr(self, 'ctx', None): - kwargs['mp_context'] = self.get_context() - executor = self.executor_type(max_workers=1, **kwargs) - executor.submit(int).result() - old_handler = signal.signal(signal.SIGALRM, timeout) - try: - signal.alarm(5) - executor.submit(int).cancel() - executor.shutdown(wait=True) - finally: - signal.alarm(0) - signal.signal(signal.SIGALRM, old_handler) - - -class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase): - def test_threads_terminate(self): - def acquire_lock(lock): - lock.acquire() - - sem = threading.Semaphore(0) - for i in range(3): - self.executor.submit(acquire_lock, sem) - self.assertEqual(len(self.executor._threads), 3) - for i in range(3): - sem.release() - self.executor.shutdown() - for t in self.executor._threads: - t.join() - - def test_context_manager_shutdown(self): - with futures.ThreadPoolExecutor(max_workers=5) as e: - executor = e - self.assertEqual(list(e.map(abs, range(-5, 5))), - [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) - - for t in executor._threads: - t.join() - - def test_del_shutdown(self): - executor = futures.ThreadPoolExecutor(max_workers=5) - res = executor.map(abs, range(-5, 5)) - threads = executor._threads - del executor - - for t in threads: - t.join() - - # Make sure the results were all computed before the - # executor got shutdown. - assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) - - def test_shutdown_no_wait(self): - # Ensure that the executor cleans up the threads when calling - # shutdown with wait=False - executor = futures.ThreadPoolExecutor(max_workers=5) - res = executor.map(abs, range(-5, 5)) - threads = executor._threads - executor.shutdown(wait=False) - for t in threads: - t.join() - - # Make sure the results were all computed before the - # executor got shutdown. - assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) - - - def test_thread_names_assigned(self): - executor = futures.ThreadPoolExecutor( - max_workers=5, thread_name_prefix='SpecialPool') - executor.map(abs, range(-5, 5)) - threads = executor._threads - del executor - support.gc_collect() # For PyPy or other GCs. - - for t in threads: - self.assertRegex(t.name, r'^SpecialPool_[0-4]$') - t.join() - - def test_thread_names_default(self): - executor = futures.ThreadPoolExecutor(max_workers=5) - executor.map(abs, range(-5, 5)) - threads = executor._threads - del executor - support.gc_collect() # For PyPy or other GCs. - - for t in threads: - # Ensure that our default name is reasonably sane and unique when - # no thread_name_prefix was supplied. - self.assertRegex(t.name, r'ThreadPoolExecutor-\d+_[0-4]$') - t.join() - - def test_cancel_futures_wait_false(self): - # Can only be reliably tested for TPE, since PPE often hangs with - # `wait=False` (even without *cancel_futures*). - rc, out, err = assert_python_ok('-c', """if True: - from concurrent.futures import ThreadPoolExecutor - from test.test_concurrent_futures import sleep_and_print - if __name__ == "__main__": - t = ThreadPoolExecutor() - t.submit(sleep_and_print, .1, "apple") - t.shutdown(wait=False, cancel_futures=True) - """) - # Errors in atexit hooks don't change the process exit code, check - # stderr manually. - self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") - - -class ProcessPoolShutdownTest(ExecutorShutdownTest): - def test_processes_terminate(self): - def acquire_lock(lock): - lock.acquire() - - mp_context = self.get_context() - if mp_context.get_start_method(allow_none=False) == "fork": - # fork pre-spawns, not on demand. - expected_num_processes = self.worker_count - else: - expected_num_processes = 3 - - sem = mp_context.Semaphore(0) - for _ in range(3): - self.executor.submit(acquire_lock, sem) - self.assertEqual(len(self.executor._processes), expected_num_processes) - for _ in range(3): - sem.release() - processes = self.executor._processes - self.executor.shutdown() - - for p in processes.values(): - p.join() - - def test_context_manager_shutdown(self): - with futures.ProcessPoolExecutor( - max_workers=5, mp_context=self.get_context()) as e: - processes = e._processes - self.assertEqual(list(e.map(abs, range(-5, 5))), - [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) - - for p in processes.values(): - p.join() - - def test_del_shutdown(self): - executor = futures.ProcessPoolExecutor( - max_workers=5, mp_context=self.get_context()) - res = executor.map(abs, range(-5, 5)) - executor_manager_thread = executor._executor_manager_thread - processes = executor._processes - call_queue = executor._call_queue - executor_manager_thread = executor._executor_manager_thread - del executor - support.gc_collect() # For PyPy or other GCs. - - # Make sure that all the executor resources were properly cleaned by - # the shutdown process - executor_manager_thread.join() - for p in processes.values(): - p.join() - call_queue.join_thread() - - # Make sure the results were all computed before the - # executor got shutdown. - assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) - - def test_shutdown_no_wait(self): - # Ensure that the executor cleans up the processes when calling - # shutdown with wait=False - executor = futures.ProcessPoolExecutor( - max_workers=5, mp_context=self.get_context()) - res = executor.map(abs, range(-5, 5)) - processes = executor._processes - call_queue = executor._call_queue - executor_manager_thread = executor._executor_manager_thread - executor.shutdown(wait=False) - - # Make sure that all the executor resources were properly cleaned by - # the shutdown process - executor_manager_thread.join() - for p in processes.values(): - p.join() - call_queue.join_thread() - - # Make sure the results were all computed before the executor got - # shutdown. - assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) - - -create_executor_tests(ProcessPoolShutdownTest, - executor_mixins=(ProcessPoolForkMixin, - ProcessPoolForkserverMixin, - ProcessPoolSpawnMixin)) - - -class WaitTests: - def test_20369(self): - # See https://bugs.python.org/issue20369 - future = self.executor.submit(time.sleep, 1.5) - done, not_done = futures.wait([future, future], - return_when=futures.ALL_COMPLETED) - self.assertEqual({future}, done) - self.assertEqual(set(), not_done) - - - def test_first_completed(self): - future1 = self.executor.submit(mul, 21, 2) - future2 = self.executor.submit(time.sleep, 1.5) - - done, not_done = futures.wait( - [CANCELLED_FUTURE, future1, future2], - return_when=futures.FIRST_COMPLETED) - - self.assertEqual(set([future1]), done) - self.assertEqual(set([CANCELLED_FUTURE, future2]), not_done) - - def test_first_completed_some_already_completed(self): - future1 = self.executor.submit(time.sleep, 1.5) - - finished, pending = futures.wait( - [CANCELLED_AND_NOTIFIED_FUTURE, SUCCESSFUL_FUTURE, future1], - return_when=futures.FIRST_COMPLETED) - - self.assertEqual( - set([CANCELLED_AND_NOTIFIED_FUTURE, SUCCESSFUL_FUTURE]), - finished) - self.assertEqual(set([future1]), pending) - - def test_first_exception(self): - future1 = self.executor.submit(mul, 2, 21) - future2 = self.executor.submit(sleep_and_raise, 1.5) - future3 = self.executor.submit(time.sleep, 3) - - finished, pending = futures.wait( - [future1, future2, future3], - return_when=futures.FIRST_EXCEPTION) - - self.assertEqual(set([future1, future2]), finished) - self.assertEqual(set([future3]), pending) - - def test_first_exception_some_already_complete(self): - future1 = self.executor.submit(divmod, 21, 0) - future2 = self.executor.submit(time.sleep, 1.5) - - finished, pending = futures.wait( - [SUCCESSFUL_FUTURE, - CANCELLED_FUTURE, - CANCELLED_AND_NOTIFIED_FUTURE, - future1, future2], - return_when=futures.FIRST_EXCEPTION) - - self.assertEqual(set([SUCCESSFUL_FUTURE, - CANCELLED_AND_NOTIFIED_FUTURE, - future1]), finished) - self.assertEqual(set([CANCELLED_FUTURE, future2]), pending) - - def test_first_exception_one_already_failed(self): - future1 = self.executor.submit(time.sleep, 2) - - finished, pending = futures.wait( - [EXCEPTION_FUTURE, future1], - return_when=futures.FIRST_EXCEPTION) - - self.assertEqual(set([EXCEPTION_FUTURE]), finished) - self.assertEqual(set([future1]), pending) - - def test_all_completed(self): - future1 = self.executor.submit(divmod, 2, 0) - future2 = self.executor.submit(mul, 2, 21) - - finished, pending = futures.wait( - [SUCCESSFUL_FUTURE, - CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - future1, - future2], - return_when=futures.ALL_COMPLETED) - - self.assertEqual(set([SUCCESSFUL_FUTURE, - CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - future1, - future2]), finished) - self.assertEqual(set(), pending) - - def test_timeout(self): - future1 = self.executor.submit(mul, 6, 7) - future2 = self.executor.submit(time.sleep, 6) - - finished, pending = futures.wait( - [CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - SUCCESSFUL_FUTURE, - future1, future2], - timeout=5, - return_when=futures.ALL_COMPLETED) - - self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - SUCCESSFUL_FUTURE, - future1]), finished) - self.assertEqual(set([future2]), pending) - - -class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): - - def test_pending_calls_race(self): - # Issue #14406: multi-threaded race condition when waiting on all - # futures. - event = threading.Event() - def future_func(): - event.wait() - oldswitchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) - try: - fs = {self.executor.submit(future_func) for i in range(100)} - event.set() - futures.wait(fs, return_when=futures.ALL_COMPLETED) - finally: - sys.setswitchinterval(oldswitchinterval) - - -create_executor_tests(WaitTests, - executor_mixins=(ProcessPoolForkMixin, - ProcessPoolForkserverMixin, - ProcessPoolSpawnMixin)) - - -class AsCompletedTests: - def test_no_timeout(self): - future1 = self.executor.submit(mul, 2, 21) - future2 = self.executor.submit(mul, 7, 6) - - completed = set(futures.as_completed( - [CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - SUCCESSFUL_FUTURE, - future1, future2])) - self.assertEqual(set( - [CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - SUCCESSFUL_FUTURE, - future1, future2]), - completed) - - def test_future_times_out(self): - """Test ``futures.as_completed`` timing out before - completing it's final future.""" - already_completed = {CANCELLED_AND_NOTIFIED_FUTURE, - EXCEPTION_FUTURE, - SUCCESSFUL_FUTURE} - - for timeout in (0, 0.01): - with self.subTest(timeout): - - future = self.executor.submit(time.sleep, 0.1) - completed_futures = set() - try: - for f in futures.as_completed( - already_completed | {future}, - timeout - ): - completed_futures.add(f) - except futures.TimeoutError: - pass - - # Check that ``future`` wasn't completed. - self.assertEqual(completed_futures, already_completed) - - def test_duplicate_futures(self): - # Issue 20367. Duplicate futures should not raise exceptions or give - # duplicate responses. - # Issue #31641: accept arbitrary iterables. - future1 = self.executor.submit(time.sleep, 2) - completed = [ - f for f in futures.as_completed(itertools.repeat(future1, 3)) - ] - self.assertEqual(len(completed), 1) - - def test_free_reference_yielded_future(self): - # Issue #14406: Generator should not keep references - # to finished futures. - futures_list = [Future() for _ in range(8)] - futures_list.append(create_future(state=CANCELLED_AND_NOTIFIED)) - futures_list.append(create_future(state=FINISHED, result=42)) - - with self.assertRaises(futures.TimeoutError): - for future in futures.as_completed(futures_list, timeout=0): - futures_list.remove(future) - wr = weakref.ref(future) - del future - support.gc_collect() # For PyPy or other GCs. - self.assertIsNone(wr()) - - futures_list[0].set_result("test") - for future in futures.as_completed(futures_list): - futures_list.remove(future) - wr = weakref.ref(future) - del future - support.gc_collect() # For PyPy or other GCs. - self.assertIsNone(wr()) - if futures_list: - futures_list[0].set_result("test") - - def test_correct_timeout_exception_msg(self): - futures_list = [CANCELLED_AND_NOTIFIED_FUTURE, PENDING_FUTURE, - RUNNING_FUTURE, SUCCESSFUL_FUTURE] - - with self.assertRaises(futures.TimeoutError) as cm: - list(futures.as_completed(futures_list, timeout=0)) - - self.assertEqual(str(cm.exception), '2 (of 4) futures unfinished') - - -create_executor_tests(AsCompletedTests) - - -class ExecutorTest: - # Executor.shutdown() and context manager usage is tested by - # ExecutorShutdownTest. - def test_submit(self): - future = self.executor.submit(pow, 2, 8) - self.assertEqual(256, future.result()) - - def test_submit_keyword(self): - future = self.executor.submit(mul, 2, y=8) - self.assertEqual(16, future.result()) - future = self.executor.submit(capture, 1, self=2, fn=3) - self.assertEqual(future.result(), ((1,), {'self': 2, 'fn': 3})) - with self.assertRaises(TypeError): - self.executor.submit(fn=capture, arg=1) - with self.assertRaises(TypeError): - self.executor.submit(arg=1) - - def test_map(self): - self.assertEqual( - list(self.executor.map(pow, range(10), range(10))), - list(map(pow, range(10), range(10)))) - - self.assertEqual( - list(self.executor.map(pow, range(10), range(10), chunksize=3)), - list(map(pow, range(10), range(10)))) - - def test_map_exception(self): - i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) - self.assertEqual(i.__next__(), (0, 1)) - self.assertEqual(i.__next__(), (0, 1)) - self.assertRaises(ZeroDivisionError, i.__next__) - - def test_map_timeout(self): - results = [] - try: - for i in self.executor.map(time.sleep, - [0, 0, 6], - timeout=5): - results.append(i) - except futures.TimeoutError: - pass - else: - self.fail('expected TimeoutError') - - self.assertEqual([None, None], results) - - def test_shutdown_race_issue12456(self): - # Issue #12456: race condition at shutdown where trying to post a - # sentinel in the call queue blocks (the queue is full while processes - # have exited). - self.executor.map(str, [2] * (self.worker_count + 1)) - self.executor.shutdown() - - @support.cpython_only - def test_no_stale_references(self): - # Issue #16284: check that the executors don't unnecessarily hang onto - # references. - my_object = MyObject() - my_object_collected = threading.Event() - my_object_callback = weakref.ref( - my_object, lambda obj: my_object_collected.set()) - # Deliberately discarding the future. - self.executor.submit(my_object.my_method) - del my_object - - collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) - self.assertTrue(collected, - "Stale reference not collected within timeout.") - - def test_max_workers_negative(self): - for number in (0, -1): - with self.assertRaisesRegex(ValueError, - "max_workers must be greater " - "than 0"): - self.executor_type(max_workers=number) - - def test_free_reference(self): - # Issue #14406: Result iterator should not keep an internal - # reference to result objects. - for obj in self.executor.map(make_dummy_object, range(10)): - wr = weakref.ref(obj) - del obj - support.gc_collect() # For PyPy or other GCs. - self.assertIsNone(wr()) - - -class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, BaseTestCase): - def test_map_submits_without_iteration(self): - """Tests verifying issue 11777.""" - finished = [] - def record_finished(n): - finished.append(n) - - self.executor.map(record_finished, range(10)) - self.executor.shutdown(wait=True) - self.assertCountEqual(finished, range(10)) - - def test_default_workers(self): - executor = self.executor_type() - expected = min(32, (os.cpu_count() or 1) + 4) - self.assertEqual(executor._max_workers, expected) - - def test_saturation(self): - executor = self.executor_type(4) - def acquire_lock(lock): - lock.acquire() - - sem = threading.Semaphore(0) - for i in range(15 * executor._max_workers): - executor.submit(acquire_lock, sem) - self.assertEqual(len(executor._threads), executor._max_workers) - for i in range(15 * executor._max_workers): - sem.release() - executor.shutdown(wait=True) - - def test_idle_thread_reuse(self): - executor = self.executor_type() - executor.submit(mul, 21, 2).result() - executor.submit(mul, 6, 7).result() - executor.submit(mul, 3, 14).result() - self.assertEqual(len(executor._threads), 1) - executor.shutdown(wait=True) - - @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') - def test_hang_global_shutdown_lock(self): - # bpo-45021: _global_shutdown_lock should be reinitialized in the child - # process, otherwise it will never exit - def submit(pool): - pool.submit(submit, pool) - - with futures.ThreadPoolExecutor(1) as pool: - pool.submit(submit, pool) - - for _ in range(50): - with futures.ProcessPoolExecutor(1, mp_context=mp.get_context('fork')) as workers: - workers.submit(tuple) - - def test_executor_map_current_future_cancel(self): - stop_event = threading.Event() - log = [] - - def log_n_wait(ident): - log.append(f"{ident=} started") - try: - stop_event.wait() - finally: - log.append(f"{ident=} stopped") - - with self.executor_type(max_workers=1) as pool: - # submit work to saturate the pool - fut = pool.submit(log_n_wait, ident="first") - try: - with contextlib.closing( - pool.map(log_n_wait, ["second", "third"], timeout=0) - ) as gen: - with self.assertRaises(TimeoutError): - next(gen) - finally: - stop_event.set() - fut.result() - # ident='second' is cancelled as a result of raising a TimeoutError - # ident='third' is cancelled because it remained in the collection of futures - self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"]) - - -class ProcessPoolExecutorTest(ExecutorTest): - - @unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit') - def test_max_workers_too_large(self): - with self.assertRaisesRegex(ValueError, - "max_workers must be <= 61"): - futures.ProcessPoolExecutor(max_workers=62) - - def test_killed_child(self): - # When a child process is abruptly terminated, the whole pool gets - # "broken". - futures = [self.executor.submit(time.sleep, 3)] - # Get one of the processes, and terminate (kill) it - p = next(iter(self.executor._processes.values())) - p.terminate() - for fut in futures: - self.assertRaises(BrokenProcessPool, fut.result) - # Submitting other jobs fails as well. - self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) - - def test_map_chunksize(self): - def bad_map(): - list(self.executor.map(pow, range(40), range(40), chunksize=-1)) - - ref = list(map(pow, range(40), range(40))) - self.assertEqual( - list(self.executor.map(pow, range(40), range(40), chunksize=6)), - ref) - self.assertEqual( - list(self.executor.map(pow, range(40), range(40), chunksize=50)), - ref) - self.assertEqual( - list(self.executor.map(pow, range(40), range(40), chunksize=40)), - ref) - self.assertRaises(ValueError, bad_map) - - @classmethod - def _test_traceback(cls): - raise RuntimeError(123) # some comment - - def test_traceback(self): - # We want ensure that the traceback from the child process is - # contained in the traceback raised in the main process. - future = self.executor.submit(self._test_traceback) - with self.assertRaises(Exception) as cm: - future.result() - - exc = cm.exception - self.assertIs(type(exc), RuntimeError) - self.assertEqual(exc.args, (123,)) - cause = exc.__cause__ - self.assertIs(type(cause), futures.process._RemoteTraceback) - self.assertIn('raise RuntimeError(123) # some comment', cause.tb) - - with support.captured_stderr() as f1: - try: - raise exc - except RuntimeError: - sys.excepthook(*sys.exc_info()) - self.assertIn('raise RuntimeError(123) # some comment', - f1.getvalue()) - - @hashlib_helper.requires_hashdigest('md5') - def test_ressources_gced_in_workers(self): - # Ensure that argument for a job are correctly gc-ed after the job - # is finished - mgr = self.get_context().Manager() - obj = EventfulGCObj(mgr) - future = self.executor.submit(id, obj) - future.result() - - self.assertTrue(obj.event.wait(timeout=1)) - - # explicitly destroy the object to ensure that EventfulGCObj.__del__() - # is called while manager is still running. - obj = None - support.gc_collect() - - mgr.shutdown() - mgr.join() - - def test_saturation(self): - executor = self.executor - mp_context = self.get_context() - sem = mp_context.Semaphore(0) - job_count = 15 * executor._max_workers - for _ in range(job_count): - executor.submit(sem.acquire) - self.assertEqual(len(executor._processes), executor._max_workers) - for _ in range(job_count): - sem.release() - - def test_idle_process_reuse_one(self): - executor = self.executor - assert executor._max_workers >= 4 - if self.get_context().get_start_method(allow_none=False) == "fork": - raise unittest.SkipTest("Incompatible with the fork start method.") - executor.submit(mul, 21, 2).result() - executor.submit(mul, 6, 7).result() - executor.submit(mul, 3, 14).result() - self.assertEqual(len(executor._processes), 1) - - def test_idle_process_reuse_multiple(self): - executor = self.executor - assert executor._max_workers <= 5 - if self.get_context().get_start_method(allow_none=False) == "fork": - raise unittest.SkipTest("Incompatible with the fork start method.") - executor.submit(mul, 12, 7).result() - executor.submit(mul, 33, 25) - executor.submit(mul, 25, 26).result() - executor.submit(mul, 18, 29) - executor.submit(mul, 1, 2).result() - executor.submit(mul, 0, 9) - self.assertLessEqual(len(executor._processes), 3) - executor.shutdown() - - def test_max_tasks_per_child(self): - context = self.get_context() - if context.get_start_method(allow_none=False) == "fork": - with self.assertRaises(ValueError): - self.executor_type(1, mp_context=context, max_tasks_per_child=3) - return - # not using self.executor as we need to control construction. - # arguably this could go in another class w/o that mixin. - executor = self.executor_type( - 1, mp_context=context, max_tasks_per_child=3) - f1 = executor.submit(os.getpid) - original_pid = f1.result() - # The worker pid remains the same as the worker could be reused - f2 = executor.submit(os.getpid) - self.assertEqual(f2.result(), original_pid) - self.assertEqual(len(executor._processes), 1) - f3 = executor.submit(os.getpid) - self.assertEqual(f3.result(), original_pid) - - # A new worker is spawned, with a statistically different pid, - # while the previous was reaped. - f4 = executor.submit(os.getpid) - new_pid = f4.result() - self.assertNotEqual(original_pid, new_pid) - self.assertEqual(len(executor._processes), 1) - - executor.shutdown() - - def test_max_tasks_per_child_defaults_to_spawn_context(self): - # not using self.executor as we need to control construction. - # arguably this could go in another class w/o that mixin. - executor = self.executor_type(1, max_tasks_per_child=3) - self.assertEqual(executor._mp_context.get_start_method(), "spawn") - - def test_max_tasks_early_shutdown(self): - context = self.get_context() - if context.get_start_method(allow_none=False) == "fork": - raise unittest.SkipTest("Incompatible with the fork start method.") - # not using self.executor as we need to control construction. - # arguably this could go in another class w/o that mixin. - executor = self.executor_type( - 3, mp_context=context, max_tasks_per_child=1) - futures = [] - for i in range(6): - futures.append(executor.submit(mul, i, i)) - executor.shutdown() - for i, future in enumerate(futures): - self.assertEqual(future.result(), mul(i, i)) - - -create_executor_tests(ProcessPoolExecutorTest, - executor_mixins=(ProcessPoolForkMixin, - ProcessPoolForkserverMixin, - ProcessPoolSpawnMixin)) - -def _crash(delay=None): - """Induces a segfault.""" - if delay: - time.sleep(delay) - import faulthandler - faulthandler.disable() - faulthandler._sigsegv() - - -def _crash_with_data(data): - """Induces a segfault with dummy data in input.""" - _crash() - - -def _exit(): - """Induces a sys exit with exitcode 1.""" - sys.exit(1) - - -def _raise_error(Err): - """Function that raises an Exception in process.""" - raise Err() - - -def _raise_error_ignore_stderr(Err): - """Function that raises an Exception in process and ignores stderr.""" - import io - sys.stderr = io.StringIO() - raise Err() - - -def _return_instance(cls): - """Function that returns a instance of cls.""" - return cls() - - -class CrashAtPickle(object): - """Bad object that triggers a segfault at pickling time.""" - def __reduce__(self): - _crash() - - -class CrashAtUnpickle(object): - """Bad object that triggers a segfault at unpickling time.""" - def __reduce__(self): - return _crash, () - - -class ExitAtPickle(object): - """Bad object that triggers a process exit at pickling time.""" - def __reduce__(self): - _exit() - - -class ExitAtUnpickle(object): - """Bad object that triggers a process exit at unpickling time.""" - def __reduce__(self): - return _exit, () - - -class ErrorAtPickle(object): - """Bad object that triggers an error at pickling time.""" - def __reduce__(self): - from pickle import PicklingError - raise PicklingError("Error in pickle") - - -class ErrorAtUnpickle(object): - """Bad object that triggers an error at unpickling time.""" - def __reduce__(self): - from pickle import UnpicklingError - return _raise_error_ignore_stderr, (UnpicklingError, ) - - -class ExecutorDeadlockTest: - TIMEOUT = support.SHORT_TIMEOUT - - def _fail_on_deadlock(self, executor): - # If we did not recover before TIMEOUT seconds, consider that the - # executor is in a deadlock state and forcefully clean all its - # composants. - import faulthandler - from tempfile import TemporaryFile - with TemporaryFile(mode="w+") as f: - faulthandler.dump_traceback(file=f) - f.seek(0) - tb = f.read() - for p in executor._processes.values(): - p.terminate() - # This should be safe to call executor.shutdown here as all possible - # deadlocks should have been broken. - executor.shutdown(wait=True) - print(f"\nTraceback:\n {tb}", file=sys.__stderr__) - self.fail(f"Executor deadlock:\n\n{tb}") - - - def _check_crash(self, error, func, *args, ignore_stderr=False): - # test for deadlock caused by crashes in a pool - self.executor.shutdown(wait=True) - - executor = self.executor_type( - max_workers=2, mp_context=self.get_context()) - res = executor.submit(func, *args) - - if ignore_stderr: - cm = support.captured_stderr() - else: - cm = contextlib.nullcontext() - - try: - with self.assertRaises(error): - with cm: - res.result(timeout=self.TIMEOUT) - except futures.TimeoutError: - # If we did not recover before TIMEOUT seconds, - # consider that the executor is in a deadlock state - self._fail_on_deadlock(executor) - executor.shutdown(wait=True) - - def test_error_at_task_pickle(self): - # Check problem occurring while pickling a task in - # the task_handler thread - self._check_crash(PicklingError, id, ErrorAtPickle()) - - def test_exit_at_task_unpickle(self): - # Check problem occurring while unpickling a task on workers - self._check_crash(BrokenProcessPool, id, ExitAtUnpickle()) - - def test_error_at_task_unpickle(self): - # Check problem occurring while unpickling a task on workers - self._check_crash(BrokenProcessPool, id, ErrorAtUnpickle()) - - def test_crash_at_task_unpickle(self): - # Check problem occurring while unpickling a task on workers - self._check_crash(BrokenProcessPool, id, CrashAtUnpickle()) - - def test_crash_during_func_exec_on_worker(self): - # Check problem occurring during func execution on workers - self._check_crash(BrokenProcessPool, _crash) - - def test_exit_during_func_exec_on_worker(self): - # Check problem occurring during func execution on workers - self._check_crash(SystemExit, _exit) - - def test_error_during_func_exec_on_worker(self): - # Check problem occurring during func execution on workers - self._check_crash(RuntimeError, _raise_error, RuntimeError) - - def test_crash_during_result_pickle_on_worker(self): - # Check problem occurring while pickling a task result - # on workers - self._check_crash(BrokenProcessPool, _return_instance, CrashAtPickle) - - def test_exit_during_result_pickle_on_worker(self): - # Check problem occurring while pickling a task result - # on workers - self._check_crash(SystemExit, _return_instance, ExitAtPickle) - - def test_error_during_result_pickle_on_worker(self): - # Check problem occurring while pickling a task result - # on workers - self._check_crash(PicklingError, _return_instance, ErrorAtPickle) - - def test_error_during_result_unpickle_in_result_handler(self): - # Check problem occurring while unpickling a task in - # the result_handler thread - self._check_crash(BrokenProcessPool, - _return_instance, ErrorAtUnpickle, - ignore_stderr=True) - - def test_exit_during_result_unpickle_in_result_handler(self): - # Check problem occurring while unpickling a task in - # the result_handler thread - self._check_crash(BrokenProcessPool, _return_instance, ExitAtUnpickle) - - def test_shutdown_deadlock(self): - # Test that the pool calling shutdown do not cause deadlock - # if a worker fails after the shutdown call. - self.executor.shutdown(wait=True) - with self.executor_type(max_workers=2, - mp_context=self.get_context()) as executor: - self.executor = executor # Allow clean up in fail_on_deadlock - f = executor.submit(_crash, delay=.1) - executor.shutdown(wait=True) - with self.assertRaises(BrokenProcessPool): - f.result() - - def test_shutdown_deadlock_pickle(self): - # Test that the pool calling shutdown with wait=False does not cause - # a deadlock if a task fails at pickle after the shutdown call. - # Reported in bpo-39104. - self.executor.shutdown(wait=True) - with self.executor_type(max_workers=2, - mp_context=self.get_context()) as executor: - self.executor = executor # Allow clean up in fail_on_deadlock - - # Start the executor and get the executor_manager_thread to collect - # the threads and avoid dangling thread that should be cleaned up - # asynchronously. - executor.submit(id, 42).result() - executor_manager = executor._executor_manager_thread - - # Submit a task that fails at pickle and shutdown the executor - # without waiting - f = executor.submit(id, ErrorAtPickle()) - executor.shutdown(wait=False) - with self.assertRaises(PicklingError): - f.result() - - # Make sure the executor is eventually shutdown and do not leave - # dangling threads - executor_manager.join() - - def test_crash_big_data(self): - # Test that there is a clean exception instad of a deadlock when a - # child process crashes while some data is being written into the - # queue. - # https://github.com/python/cpython/issues/94777 - self.executor.shutdown(wait=True) - data = "a" * support.PIPE_MAX_SIZE - with self.executor_type(max_workers=2, - mp_context=self.get_context()) as executor: - self.executor = executor # Allow clean up in fail_on_deadlock - with self.assertRaises(BrokenProcessPool): - list(executor.map(_crash_with_data, [data] * 10)) - - -create_executor_tests(ExecutorDeadlockTest, - executor_mixins=(ProcessPoolForkMixin, - ProcessPoolForkserverMixin, - ProcessPoolSpawnMixin)) - - -class FutureTests(BaseTestCase): - def test_done_callback_with_result(self): - callback_result = None - def fn(callback_future): - nonlocal callback_result - callback_result = callback_future.result() - - f = Future() - f.add_done_callback(fn) - f.set_result(5) - self.assertEqual(5, callback_result) - - def test_done_callback_with_exception(self): - callback_exception = None - def fn(callback_future): - nonlocal callback_exception - callback_exception = callback_future.exception() - - f = Future() - f.add_done_callback(fn) - f.set_exception(Exception('test')) - self.assertEqual(('test',), callback_exception.args) - - def test_done_callback_with_cancel(self): - was_cancelled = None - def fn(callback_future): - nonlocal was_cancelled - was_cancelled = callback_future.cancelled() - - f = Future() - f.add_done_callback(fn) - self.assertTrue(f.cancel()) - self.assertTrue(was_cancelled) - - def test_done_callback_raises(self): - with support.captured_stderr() as stderr: - raising_was_called = False - fn_was_called = False - - def raising_fn(callback_future): - nonlocal raising_was_called - raising_was_called = True - raise Exception('doh!') - - def fn(callback_future): - nonlocal fn_was_called - fn_was_called = True - - f = Future() - f.add_done_callback(raising_fn) - f.add_done_callback(fn) - f.set_result(5) - self.assertTrue(raising_was_called) - self.assertTrue(fn_was_called) - self.assertIn('Exception: doh!', stderr.getvalue()) - - def test_done_callback_already_successful(self): - callback_result = None - def fn(callback_future): - nonlocal callback_result - callback_result = callback_future.result() - - f = Future() - f.set_result(5) - f.add_done_callback(fn) - self.assertEqual(5, callback_result) - - def test_done_callback_already_failed(self): - callback_exception = None - def fn(callback_future): - nonlocal callback_exception - callback_exception = callback_future.exception() - - f = Future() - f.set_exception(Exception('test')) - f.add_done_callback(fn) - self.assertEqual(('test',), callback_exception.args) - - def test_done_callback_already_cancelled(self): - was_cancelled = None - def fn(callback_future): - nonlocal was_cancelled - was_cancelled = callback_future.cancelled() - - f = Future() - self.assertTrue(f.cancel()) - f.add_done_callback(fn) - self.assertTrue(was_cancelled) - - def test_done_callback_raises_already_succeeded(self): - with support.captured_stderr() as stderr: - def raising_fn(callback_future): - raise Exception('doh!') - - f = Future() - - # Set the result first to simulate a future that runs instantly, - # effectively allowing the callback to be run immediately. - f.set_result(5) - f.add_done_callback(raising_fn) - - self.assertIn('exception calling callback for', stderr.getvalue()) - self.assertIn('doh!', stderr.getvalue()) - - - def test_repr(self): - self.assertRegex(repr(PENDING_FUTURE), - '') - self.assertRegex(repr(RUNNING_FUTURE), - '') - self.assertRegex(repr(CANCELLED_FUTURE), - '') - self.assertRegex(repr(CANCELLED_AND_NOTIFIED_FUTURE), - '') - self.assertRegex( - repr(EXCEPTION_FUTURE), - '') - self.assertRegex( - repr(SUCCESSFUL_FUTURE), - '') - - - def test_cancel(self): - f1 = create_future(state=PENDING) - f2 = create_future(state=RUNNING) - f3 = create_future(state=CANCELLED) - f4 = create_future(state=CANCELLED_AND_NOTIFIED) - f5 = create_future(state=FINISHED, exception=OSError()) - f6 = create_future(state=FINISHED, result=5) - - self.assertTrue(f1.cancel()) - self.assertEqual(f1._state, CANCELLED) - - self.assertFalse(f2.cancel()) - self.assertEqual(f2._state, RUNNING) - - self.assertTrue(f3.cancel()) - self.assertEqual(f3._state, CANCELLED) - - self.assertTrue(f4.cancel()) - self.assertEqual(f4._state, CANCELLED_AND_NOTIFIED) - - self.assertFalse(f5.cancel()) - self.assertEqual(f5._state, FINISHED) - - self.assertFalse(f6.cancel()) - self.assertEqual(f6._state, FINISHED) - - def test_cancelled(self): - self.assertFalse(PENDING_FUTURE.cancelled()) - self.assertFalse(RUNNING_FUTURE.cancelled()) - self.assertTrue(CANCELLED_FUTURE.cancelled()) - self.assertTrue(CANCELLED_AND_NOTIFIED_FUTURE.cancelled()) - self.assertFalse(EXCEPTION_FUTURE.cancelled()) - self.assertFalse(SUCCESSFUL_FUTURE.cancelled()) - - def test_done(self): - self.assertFalse(PENDING_FUTURE.done()) - self.assertFalse(RUNNING_FUTURE.done()) - self.assertTrue(CANCELLED_FUTURE.done()) - self.assertTrue(CANCELLED_AND_NOTIFIED_FUTURE.done()) - self.assertTrue(EXCEPTION_FUTURE.done()) - self.assertTrue(SUCCESSFUL_FUTURE.done()) - - def test_running(self): - self.assertFalse(PENDING_FUTURE.running()) - self.assertTrue(RUNNING_FUTURE.running()) - self.assertFalse(CANCELLED_FUTURE.running()) - self.assertFalse(CANCELLED_AND_NOTIFIED_FUTURE.running()) - self.assertFalse(EXCEPTION_FUTURE.running()) - self.assertFalse(SUCCESSFUL_FUTURE.running()) - - def test_result_with_timeout(self): - self.assertRaises(futures.TimeoutError, - PENDING_FUTURE.result, timeout=0) - self.assertRaises(futures.TimeoutError, - RUNNING_FUTURE.result, timeout=0) - self.assertRaises(futures.CancelledError, - CANCELLED_FUTURE.result, timeout=0) - self.assertRaises(futures.CancelledError, - CANCELLED_AND_NOTIFIED_FUTURE.result, timeout=0) - self.assertRaises(OSError, EXCEPTION_FUTURE.result, timeout=0) - self.assertEqual(SUCCESSFUL_FUTURE.result(timeout=0), 42) - - def test_result_with_success(self): - # TODO(brian@sweetapp.com): This test is timing dependent. - def notification(): - # Wait until the main thread is waiting for the result. - time.sleep(1) - f1.set_result(42) - - f1 = create_future(state=PENDING) - t = threading.Thread(target=notification) - t.start() - - self.assertEqual(f1.result(timeout=5), 42) - t.join() - - def test_result_with_cancel(self): - # TODO(brian@sweetapp.com): This test is timing dependent. - def notification(): - # Wait until the main thread is waiting for the result. - time.sleep(1) - f1.cancel() - - f1 = create_future(state=PENDING) - t = threading.Thread(target=notification) - t.start() - - self.assertRaises(futures.CancelledError, - f1.result, timeout=support.SHORT_TIMEOUT) - t.join() - - def test_exception_with_timeout(self): - self.assertRaises(futures.TimeoutError, - PENDING_FUTURE.exception, timeout=0) - self.assertRaises(futures.TimeoutError, - RUNNING_FUTURE.exception, timeout=0) - self.assertRaises(futures.CancelledError, - CANCELLED_FUTURE.exception, timeout=0) - self.assertRaises(futures.CancelledError, - CANCELLED_AND_NOTIFIED_FUTURE.exception, timeout=0) - self.assertTrue(isinstance(EXCEPTION_FUTURE.exception(timeout=0), - OSError)) - self.assertEqual(SUCCESSFUL_FUTURE.exception(timeout=0), None) - - def test_exception_with_success(self): - def notification(): - # Wait until the main thread is waiting for the exception. - time.sleep(1) - with f1._condition: - f1._state = FINISHED - f1._exception = OSError() - f1._condition.notify_all() - - f1 = create_future(state=PENDING) - t = threading.Thread(target=notification) - t.start() - - self.assertTrue(isinstance(f1.exception(timeout=support.SHORT_TIMEOUT), OSError)) - t.join() - - def test_multiple_set_result(self): - f = create_future(state=PENDING) - f.set_result(1) - - with self.assertRaisesRegex( - futures.InvalidStateError, - 'FINISHED: ' - ): - f.set_result(2) - - self.assertTrue(f.done()) - self.assertEqual(f.result(), 1) - - def test_multiple_set_exception(self): - f = create_future(state=PENDING) - e = ValueError() - f.set_exception(e) - - with self.assertRaisesRegex( - futures.InvalidStateError, - 'FINISHED: ' - ): - f.set_exception(Exception()) - - self.assertEqual(f.exception(), e) - - -def setUpModule(): - unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) - thread_info = threading_helper.threading_setup() - unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_concurrent_futures/__init__.py b/Lib/test/test_concurrent_futures/__init__.py new file mode 100644 index 00000000000000..430fa93aa456a2 --- /dev/null +++ b/Lib/test/test_concurrent_futures/__init__.py @@ -0,0 +1,16 @@ +import os.path +import unittest +from test import support +from test.support import import_helper + +# Skip tests if _multiprocessing wasn't built. +import_helper.import_module('_multiprocessing') + +if support.check_sanitizer(address=True, memory=True): + # gh-90791: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py new file mode 100644 index 00000000000000..36278bdd501971 --- /dev/null +++ b/Lib/test/test_concurrent_futures/executor.py @@ -0,0 +1,107 @@ +import threading +import time +import weakref +from concurrent import futures +from test import support + + +def mul(x, y): + return x * y + +def capture(*args, **kwargs): + return args, kwargs + + +class MyObject(object): + def my_method(self): + pass + + +def make_dummy_object(_): + return MyObject() + + +class ExecutorTest: + # Executor.shutdown() and context manager usage is tested by + # ExecutorShutdownTest. + def test_submit(self): + future = self.executor.submit(pow, 2, 8) + self.assertEqual(256, future.result()) + + def test_submit_keyword(self): + future = self.executor.submit(mul, 2, y=8) + self.assertEqual(16, future.result()) + future = self.executor.submit(capture, 1, self=2, fn=3) + self.assertEqual(future.result(), ((1,), {'self': 2, 'fn': 3})) + with self.assertRaises(TypeError): + self.executor.submit(fn=capture, arg=1) + with self.assertRaises(TypeError): + self.executor.submit(arg=1) + + def test_map(self): + self.assertEqual( + list(self.executor.map(pow, range(10), range(10))), + list(map(pow, range(10), range(10)))) + + self.assertEqual( + list(self.executor.map(pow, range(10), range(10), chunksize=3)), + list(map(pow, range(10), range(10)))) + + def test_map_exception(self): + i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) + self.assertEqual(i.__next__(), (0, 1)) + self.assertEqual(i.__next__(), (0, 1)) + self.assertRaises(ZeroDivisionError, i.__next__) + + def test_map_timeout(self): + results = [] + try: + for i in self.executor.map(time.sleep, + [0, 0, 6], + timeout=5): + results.append(i) + except futures.TimeoutError: + pass + else: + self.fail('expected TimeoutError') + + self.assertEqual([None, None], results) + + def test_shutdown_race_issue12456(self): + # Issue #12456: race condition at shutdown where trying to post a + # sentinel in the call queue blocks (the queue is full while processes + # have exited). + self.executor.map(str, [2] * (self.worker_count + 1)) + self.executor.shutdown() + + @support.cpython_only + def test_no_stale_references(self): + # Issue #16284: check that the executors don't unnecessarily hang onto + # references. + my_object = MyObject() + my_object_collected = threading.Event() + my_object_callback = weakref.ref( + my_object, lambda obj: my_object_collected.set()) + # Deliberately discarding the future. + self.executor.submit(my_object.my_method) + del my_object + + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) + self.assertTrue(collected, + "Stale reference not collected within timeout.") + + def test_max_workers_negative(self): + for number in (0, -1): + with self.assertRaisesRegex(ValueError, + "max_workers must be greater " + "than 0"): + self.executor_type(max_workers=number) + + def test_free_reference(self): + # Issue #14406: Result iterator should not keep an internal + # reference to result objects. + for obj in self.executor.map(make_dummy_object, range(10)): + wr = weakref.ref(obj) + del obj + support.gc_collect() # For PyPy or other GCs. + self.assertIsNone(wr()) diff --git a/Lib/test/test_concurrent_futures/test_as_completed.py b/Lib/test/test_concurrent_futures/test_as_completed.py new file mode 100644 index 00000000000000..2b3bec8cafbcb0 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_as_completed.py @@ -0,0 +1,115 @@ +import itertools +import time +import unittest +import weakref +from concurrent import futures +from concurrent.futures._base import ( + CANCELLED_AND_NOTIFIED, FINISHED, Future) + +from test import support + +from .util import ( + PENDING_FUTURE, RUNNING_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, + create_future, create_executor_tests, setup_module) + + +def mul(x, y): + return x * y + + +class AsCompletedTests: + def test_no_timeout(self): + future1 = self.executor.submit(mul, 2, 21) + future2 = self.executor.submit(mul, 7, 6) + + completed = set(futures.as_completed( + [CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future1, future2])) + self.assertEqual(set( + [CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future1, future2]), + completed) + + def test_future_times_out(self): + """Test ``futures.as_completed`` timing out before + completing it's final future.""" + already_completed = {CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE} + + for timeout in (0, 0.01): + with self.subTest(timeout): + + future = self.executor.submit(time.sleep, 0.1) + completed_futures = set() + try: + for f in futures.as_completed( + already_completed | {future}, + timeout + ): + completed_futures.add(f) + except futures.TimeoutError: + pass + + # Check that ``future`` wasn't completed. + self.assertEqual(completed_futures, already_completed) + + def test_duplicate_futures(self): + # Issue 20367. Duplicate futures should not raise exceptions or give + # duplicate responses. + # Issue #31641: accept arbitrary iterables. + future1 = self.executor.submit(time.sleep, 2) + completed = [ + f for f in futures.as_completed(itertools.repeat(future1, 3)) + ] + self.assertEqual(len(completed), 1) + + def test_free_reference_yielded_future(self): + # Issue #14406: Generator should not keep references + # to finished futures. + futures_list = [Future() for _ in range(8)] + futures_list.append(create_future(state=CANCELLED_AND_NOTIFIED)) + futures_list.append(create_future(state=FINISHED, result=42)) + + with self.assertRaises(futures.TimeoutError): + for future in futures.as_completed(futures_list, timeout=0): + futures_list.remove(future) + wr = weakref.ref(future) + del future + support.gc_collect() # For PyPy or other GCs. + self.assertIsNone(wr()) + + futures_list[0].set_result("test") + for future in futures.as_completed(futures_list): + futures_list.remove(future) + wr = weakref.ref(future) + del future + support.gc_collect() # For PyPy or other GCs. + self.assertIsNone(wr()) + if futures_list: + futures_list[0].set_result("test") + + def test_correct_timeout_exception_msg(self): + futures_list = [CANCELLED_AND_NOTIFIED_FUTURE, PENDING_FUTURE, + RUNNING_FUTURE, SUCCESSFUL_FUTURE] + + with self.assertRaises(futures.TimeoutError) as cm: + list(futures.as_completed(futures_list, timeout=0)) + + self.assertEqual(str(cm.exception), '2 (of 4) futures unfinished') + + +create_executor_tests(globals(), AsCompletedTests) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py new file mode 100644 index 00000000000000..6b78b360d15627 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -0,0 +1,253 @@ +import contextlib +import sys +import time +import unittest +from pickle import PicklingError +from concurrent import futures +from concurrent.futures.process import BrokenProcessPool + +from test import support + +from .util import ( + create_executor_tests, setup_module, + ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin) + + +def _crash(delay=None): + """Induces a segfault.""" + if delay: + time.sleep(delay) + import faulthandler + faulthandler.disable() + faulthandler._sigsegv() + + +def _crash_with_data(data): + """Induces a segfault with dummy data in input.""" + _crash() + + +def _exit(): + """Induces a sys exit with exitcode 1.""" + sys.exit(1) + + +def _raise_error(Err): + """Function that raises an Exception in process.""" + raise Err() + + +def _raise_error_ignore_stderr(Err): + """Function that raises an Exception in process and ignores stderr.""" + import io + sys.stderr = io.StringIO() + raise Err() + + +def _return_instance(cls): + """Function that returns a instance of cls.""" + return cls() + + +class CrashAtPickle(object): + """Bad object that triggers a segfault at pickling time.""" + def __reduce__(self): + _crash() + + +class CrashAtUnpickle(object): + """Bad object that triggers a segfault at unpickling time.""" + def __reduce__(self): + return _crash, () + + +class ExitAtPickle(object): + """Bad object that triggers a process exit at pickling time.""" + def __reduce__(self): + _exit() + + +class ExitAtUnpickle(object): + """Bad object that triggers a process exit at unpickling time.""" + def __reduce__(self): + return _exit, () + + +class ErrorAtPickle(object): + """Bad object that triggers an error at pickling time.""" + def __reduce__(self): + from pickle import PicklingError + raise PicklingError("Error in pickle") + + +class ErrorAtUnpickle(object): + """Bad object that triggers an error at unpickling time.""" + def __reduce__(self): + from pickle import UnpicklingError + return _raise_error_ignore_stderr, (UnpicklingError, ) + + +class ExecutorDeadlockTest: + TIMEOUT = support.SHORT_TIMEOUT + + def _fail_on_deadlock(self, executor): + # If we did not recover before TIMEOUT seconds, consider that the + # executor is in a deadlock state and forcefully clean all its + # composants. + import faulthandler + from tempfile import TemporaryFile + with TemporaryFile(mode="w+") as f: + faulthandler.dump_traceback(file=f) + f.seek(0) + tb = f.read() + for p in executor._processes.values(): + p.terminate() + # This should be safe to call executor.shutdown here as all possible + # deadlocks should have been broken. + executor.shutdown(wait=True) + print(f"\nTraceback:\n {tb}", file=sys.__stderr__) + self.fail(f"Executor deadlock:\n\n{tb}") + + + def _check_crash(self, error, func, *args, ignore_stderr=False): + # test for deadlock caused by crashes in a pool + self.executor.shutdown(wait=True) + + executor = self.executor_type( + max_workers=2, mp_context=self.get_context()) + res = executor.submit(func, *args) + + if ignore_stderr: + cm = support.captured_stderr() + else: + cm = contextlib.nullcontext() + + try: + with self.assertRaises(error): + with cm: + res.result(timeout=self.TIMEOUT) + except futures.TimeoutError: + # If we did not recover before TIMEOUT seconds, + # consider that the executor is in a deadlock state + self._fail_on_deadlock(executor) + executor.shutdown(wait=True) + + def test_error_at_task_pickle(self): + # Check problem occurring while pickling a task in + # the task_handler thread + self._check_crash(PicklingError, id, ErrorAtPickle()) + + def test_exit_at_task_unpickle(self): + # Check problem occurring while unpickling a task on workers + self._check_crash(BrokenProcessPool, id, ExitAtUnpickle()) + + def test_error_at_task_unpickle(self): + # Check problem occurring while unpickling a task on workers + self._check_crash(BrokenProcessPool, id, ErrorAtUnpickle()) + + def test_crash_at_task_unpickle(self): + # Check problem occurring while unpickling a task on workers + self._check_crash(BrokenProcessPool, id, CrashAtUnpickle()) + + def test_crash_during_func_exec_on_worker(self): + # Check problem occurring during func execution on workers + self._check_crash(BrokenProcessPool, _crash) + + def test_exit_during_func_exec_on_worker(self): + # Check problem occurring during func execution on workers + self._check_crash(SystemExit, _exit) + + def test_error_during_func_exec_on_worker(self): + # Check problem occurring during func execution on workers + self._check_crash(RuntimeError, _raise_error, RuntimeError) + + def test_crash_during_result_pickle_on_worker(self): + # Check problem occurring while pickling a task result + # on workers + self._check_crash(BrokenProcessPool, _return_instance, CrashAtPickle) + + def test_exit_during_result_pickle_on_worker(self): + # Check problem occurring while pickling a task result + # on workers + self._check_crash(SystemExit, _return_instance, ExitAtPickle) + + def test_error_during_result_pickle_on_worker(self): + # Check problem occurring while pickling a task result + # on workers + self._check_crash(PicklingError, _return_instance, ErrorAtPickle) + + def test_error_during_result_unpickle_in_result_handler(self): + # Check problem occurring while unpickling a task in + # the result_handler thread + self._check_crash(BrokenProcessPool, + _return_instance, ErrorAtUnpickle, + ignore_stderr=True) + + def test_exit_during_result_unpickle_in_result_handler(self): + # Check problem occurring while unpickling a task in + # the result_handler thread + self._check_crash(BrokenProcessPool, _return_instance, ExitAtUnpickle) + + def test_shutdown_deadlock(self): + # Test that the pool calling shutdown do not cause deadlock + # if a worker fails after the shutdown call. + self.executor.shutdown(wait=True) + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + f = executor.submit(_crash, delay=.1) + executor.shutdown(wait=True) + with self.assertRaises(BrokenProcessPool): + f.result() + + def test_shutdown_deadlock_pickle(self): + # Test that the pool calling shutdown with wait=False does not cause + # a deadlock if a task fails at pickle after the shutdown call. + # Reported in bpo-39104. + self.executor.shutdown(wait=True) + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + + # Start the executor and get the executor_manager_thread to collect + # the threads and avoid dangling thread that should be cleaned up + # asynchronously. + executor.submit(id, 42).result() + executor_manager = executor._executor_manager_thread + + # Submit a task that fails at pickle and shutdown the executor + # without waiting + f = executor.submit(id, ErrorAtPickle()) + executor.shutdown(wait=False) + with self.assertRaises(PicklingError): + f.result() + + # Make sure the executor is eventually shutdown and do not leave + # dangling threads + executor_manager.join() + + def test_crash_big_data(self): + # Test that there is a clean exception instad of a deadlock when a + # child process crashes while some data is being written into the + # queue. + # https://github.com/python/cpython/issues/94777 + self.executor.shutdown(wait=True) + data = "a" * support.PIPE_MAX_SIZE + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + with self.assertRaises(BrokenProcessPool): + list(executor.map(_crash_with_data, [data] * 10)) + + +create_executor_tests(globals(), ExecutorDeadlockTest, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_future.py b/Lib/test/test_concurrent_futures/test_future.py new file mode 100644 index 00000000000000..4066ea1ee4b367 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_future.py @@ -0,0 +1,291 @@ +import threading +import time +import unittest +from concurrent import futures +from concurrent.futures._base import ( + PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future) + +from test import support + +from .util import ( + PENDING_FUTURE, RUNNING_FUTURE, CANCELLED_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, + BaseTestCase, create_future, setup_module) + + +class FutureTests(BaseTestCase): + def test_done_callback_with_result(self): + callback_result = None + def fn(callback_future): + nonlocal callback_result + callback_result = callback_future.result() + + f = Future() + f.add_done_callback(fn) + f.set_result(5) + self.assertEqual(5, callback_result) + + def test_done_callback_with_exception(self): + callback_exception = None + def fn(callback_future): + nonlocal callback_exception + callback_exception = callback_future.exception() + + f = Future() + f.add_done_callback(fn) + f.set_exception(Exception('test')) + self.assertEqual(('test',), callback_exception.args) + + def test_done_callback_with_cancel(self): + was_cancelled = None + def fn(callback_future): + nonlocal was_cancelled + was_cancelled = callback_future.cancelled() + + f = Future() + f.add_done_callback(fn) + self.assertTrue(f.cancel()) + self.assertTrue(was_cancelled) + + def test_done_callback_raises(self): + with support.captured_stderr() as stderr: + raising_was_called = False + fn_was_called = False + + def raising_fn(callback_future): + nonlocal raising_was_called + raising_was_called = True + raise Exception('doh!') + + def fn(callback_future): + nonlocal fn_was_called + fn_was_called = True + + f = Future() + f.add_done_callback(raising_fn) + f.add_done_callback(fn) + f.set_result(5) + self.assertTrue(raising_was_called) + self.assertTrue(fn_was_called) + self.assertIn('Exception: doh!', stderr.getvalue()) + + def test_done_callback_already_successful(self): + callback_result = None + def fn(callback_future): + nonlocal callback_result + callback_result = callback_future.result() + + f = Future() + f.set_result(5) + f.add_done_callback(fn) + self.assertEqual(5, callback_result) + + def test_done_callback_already_failed(self): + callback_exception = None + def fn(callback_future): + nonlocal callback_exception + callback_exception = callback_future.exception() + + f = Future() + f.set_exception(Exception('test')) + f.add_done_callback(fn) + self.assertEqual(('test',), callback_exception.args) + + def test_done_callback_already_cancelled(self): + was_cancelled = None + def fn(callback_future): + nonlocal was_cancelled + was_cancelled = callback_future.cancelled() + + f = Future() + self.assertTrue(f.cancel()) + f.add_done_callback(fn) + self.assertTrue(was_cancelled) + + def test_done_callback_raises_already_succeeded(self): + with support.captured_stderr() as stderr: + def raising_fn(callback_future): + raise Exception('doh!') + + f = Future() + + # Set the result first to simulate a future that runs instantly, + # effectively allowing the callback to be run immediately. + f.set_result(5) + f.add_done_callback(raising_fn) + + self.assertIn('exception calling callback for', stderr.getvalue()) + self.assertIn('doh!', stderr.getvalue()) + + + def test_repr(self): + self.assertRegex(repr(PENDING_FUTURE), + '') + self.assertRegex(repr(RUNNING_FUTURE), + '') + self.assertRegex(repr(CANCELLED_FUTURE), + '') + self.assertRegex(repr(CANCELLED_AND_NOTIFIED_FUTURE), + '') + self.assertRegex( + repr(EXCEPTION_FUTURE), + '') + self.assertRegex( + repr(SUCCESSFUL_FUTURE), + '') + + def test_cancel(self): + f1 = create_future(state=PENDING) + f2 = create_future(state=RUNNING) + f3 = create_future(state=CANCELLED) + f4 = create_future(state=CANCELLED_AND_NOTIFIED) + f5 = create_future(state=FINISHED, exception=OSError()) + f6 = create_future(state=FINISHED, result=5) + + self.assertTrue(f1.cancel()) + self.assertEqual(f1._state, CANCELLED) + + self.assertFalse(f2.cancel()) + self.assertEqual(f2._state, RUNNING) + + self.assertTrue(f3.cancel()) + self.assertEqual(f3._state, CANCELLED) + + self.assertTrue(f4.cancel()) + self.assertEqual(f4._state, CANCELLED_AND_NOTIFIED) + + self.assertFalse(f5.cancel()) + self.assertEqual(f5._state, FINISHED) + + self.assertFalse(f6.cancel()) + self.assertEqual(f6._state, FINISHED) + + def test_cancelled(self): + self.assertFalse(PENDING_FUTURE.cancelled()) + self.assertFalse(RUNNING_FUTURE.cancelled()) + self.assertTrue(CANCELLED_FUTURE.cancelled()) + self.assertTrue(CANCELLED_AND_NOTIFIED_FUTURE.cancelled()) + self.assertFalse(EXCEPTION_FUTURE.cancelled()) + self.assertFalse(SUCCESSFUL_FUTURE.cancelled()) + + def test_done(self): + self.assertFalse(PENDING_FUTURE.done()) + self.assertFalse(RUNNING_FUTURE.done()) + self.assertTrue(CANCELLED_FUTURE.done()) + self.assertTrue(CANCELLED_AND_NOTIFIED_FUTURE.done()) + self.assertTrue(EXCEPTION_FUTURE.done()) + self.assertTrue(SUCCESSFUL_FUTURE.done()) + + def test_running(self): + self.assertFalse(PENDING_FUTURE.running()) + self.assertTrue(RUNNING_FUTURE.running()) + self.assertFalse(CANCELLED_FUTURE.running()) + self.assertFalse(CANCELLED_AND_NOTIFIED_FUTURE.running()) + self.assertFalse(EXCEPTION_FUTURE.running()) + self.assertFalse(SUCCESSFUL_FUTURE.running()) + + def test_result_with_timeout(self): + self.assertRaises(futures.TimeoutError, + PENDING_FUTURE.result, timeout=0) + self.assertRaises(futures.TimeoutError, + RUNNING_FUTURE.result, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_FUTURE.result, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_AND_NOTIFIED_FUTURE.result, timeout=0) + self.assertRaises(OSError, EXCEPTION_FUTURE.result, timeout=0) + self.assertEqual(SUCCESSFUL_FUTURE.result(timeout=0), 42) + + def test_result_with_success(self): + # TODO(brian@sweetapp.com): This test is timing dependent. + def notification(): + # Wait until the main thread is waiting for the result. + time.sleep(1) + f1.set_result(42) + + f1 = create_future(state=PENDING) + t = threading.Thread(target=notification) + t.start() + + self.assertEqual(f1.result(timeout=5), 42) + t.join() + + def test_result_with_cancel(self): + # TODO(brian@sweetapp.com): This test is timing dependent. + def notification(): + # Wait until the main thread is waiting for the result. + time.sleep(1) + f1.cancel() + + f1 = create_future(state=PENDING) + t = threading.Thread(target=notification) + t.start() + + self.assertRaises(futures.CancelledError, + f1.result, timeout=support.SHORT_TIMEOUT) + t.join() + + def test_exception_with_timeout(self): + self.assertRaises(futures.TimeoutError, + PENDING_FUTURE.exception, timeout=0) + self.assertRaises(futures.TimeoutError, + RUNNING_FUTURE.exception, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_FUTURE.exception, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_AND_NOTIFIED_FUTURE.exception, timeout=0) + self.assertTrue(isinstance(EXCEPTION_FUTURE.exception(timeout=0), + OSError)) + self.assertEqual(SUCCESSFUL_FUTURE.exception(timeout=0), None) + + def test_exception_with_success(self): + def notification(): + # Wait until the main thread is waiting for the exception. + time.sleep(1) + with f1._condition: + f1._state = FINISHED + f1._exception = OSError() + f1._condition.notify_all() + + f1 = create_future(state=PENDING) + t = threading.Thread(target=notification) + t.start() + + self.assertTrue(isinstance(f1.exception(timeout=support.SHORT_TIMEOUT), OSError)) + t.join() + + def test_multiple_set_result(self): + f = create_future(state=PENDING) + f.set_result(1) + + with self.assertRaisesRegex( + futures.InvalidStateError, + 'FINISHED: ' + ): + f.set_result(2) + + self.assertTrue(f.done()) + self.assertEqual(f.result(), 1) + + def test_multiple_set_exception(self): + f = create_future(state=PENDING) + e = ValueError() + f.set_exception(e) + + with self.assertRaisesRegex( + futures.InvalidStateError, + 'FINISHED: ' + ): + f.set_exception(Exception()) + + self.assertEqual(f.exception(), e) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py new file mode 100644 index 00000000000000..ce01e0ff0f287a --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -0,0 +1,117 @@ +import contextlib +import logging +import queue +import time +import unittest +from concurrent.futures._base import BrokenExecutor +from logging.handlers import QueueHandler + +from test import support + +from .util import ExecutorMixin, create_executor_tests, setup_module + + +INITIALIZER_STATUS = 'uninitialized' + +def init(x): + global INITIALIZER_STATUS + INITIALIZER_STATUS = x + +def get_init_status(): + return INITIALIZER_STATUS + +def init_fail(log_queue=None): + if log_queue is not None: + logger = logging.getLogger('concurrent.futures') + logger.addHandler(QueueHandler(log_queue)) + logger.setLevel('CRITICAL') + logger.propagate = False + time.sleep(0.1) # let some futures be scheduled + raise ValueError('error in initializer') + + +class InitializerMixin(ExecutorMixin): + worker_count = 2 + + def setUp(self): + global INITIALIZER_STATUS + INITIALIZER_STATUS = 'uninitialized' + self.executor_kwargs = dict(initializer=init, + initargs=('initialized',)) + super().setUp() + + def test_initializer(self): + futures = [self.executor.submit(get_init_status) + for _ in range(self.worker_count)] + + for f in futures: + self.assertEqual(f.result(), 'initialized') + + +class FailingInitializerMixin(ExecutorMixin): + worker_count = 2 + + def setUp(self): + if hasattr(self, "ctx"): + # Pass a queue to redirect the child's logging output + self.mp_context = self.get_context() + self.log_queue = self.mp_context.Queue() + self.executor_kwargs = dict(initializer=init_fail, + initargs=(self.log_queue,)) + else: + # In a thread pool, the child shares our logging setup + # (see _assert_logged()) + self.mp_context = None + self.log_queue = None + self.executor_kwargs = dict(initializer=init_fail) + super().setUp() + + def test_initializer(self): + with self._assert_logged('ValueError: error in initializer'): + try: + future = self.executor.submit(get_init_status) + except BrokenExecutor: + # Perhaps the executor is already broken + pass + else: + with self.assertRaises(BrokenExecutor): + future.result() + + # At some point, the executor should break + for _ in support.sleeping_retry(support.SHORT_TIMEOUT, + "executor not broken"): + if self.executor._broken: + break + + # ... and from this point submit() is guaranteed to fail + with self.assertRaises(BrokenExecutor): + self.executor.submit(get_init_status) + + @contextlib.contextmanager + def _assert_logged(self, msg): + if self.log_queue is not None: + yield + output = [] + try: + while True: + output.append(self.log_queue.get_nowait().getMessage()) + except queue.Empty: + pass + else: + with self.assertLogs('concurrent.futures', 'CRITICAL') as cm: + yield + output = cm.output + self.assertTrue(any(msg in line for line in output), + output) + + +create_executor_tests(globals(), InitializerMixin) +create_executor_tests(globals(), FailingInitializerMixin) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py new file mode 100644 index 00000000000000..7763a4946f110c --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -0,0 +1,202 @@ +import os +import sys +import time +import unittest +from concurrent import futures +from concurrent.futures.process import BrokenProcessPool + +from test import support +from test.support import hashlib_helper + +from .executor import ExecutorTest, mul +from .util import ( + ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin, + create_executor_tests, setup_module) + + +class EventfulGCObj(): + def __init__(self, mgr): + self.event = mgr.Event() + + def __del__(self): + self.event.set() + + +class ProcessPoolExecutorTest(ExecutorTest): + + @unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit') + def test_max_workers_too_large(self): + with self.assertRaisesRegex(ValueError, + "max_workers must be <= 61"): + futures.ProcessPoolExecutor(max_workers=62) + + def test_killed_child(self): + # When a child process is abruptly terminated, the whole pool gets + # "broken". + futures = [self.executor.submit(time.sleep, 3)] + # Get one of the processes, and terminate (kill) it + p = next(iter(self.executor._processes.values())) + p.terminate() + for fut in futures: + self.assertRaises(BrokenProcessPool, fut.result) + # Submitting other jobs fails as well. + self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) + + def test_map_chunksize(self): + def bad_map(): + list(self.executor.map(pow, range(40), range(40), chunksize=-1)) + + ref = list(map(pow, range(40), range(40))) + self.assertEqual( + list(self.executor.map(pow, range(40), range(40), chunksize=6)), + ref) + self.assertEqual( + list(self.executor.map(pow, range(40), range(40), chunksize=50)), + ref) + self.assertEqual( + list(self.executor.map(pow, range(40), range(40), chunksize=40)), + ref) + self.assertRaises(ValueError, bad_map) + + @classmethod + def _test_traceback(cls): + raise RuntimeError(123) # some comment + + def test_traceback(self): + # We want ensure that the traceback from the child process is + # contained in the traceback raised in the main process. + future = self.executor.submit(self._test_traceback) + with self.assertRaises(Exception) as cm: + future.result() + + exc = cm.exception + self.assertIs(type(exc), RuntimeError) + self.assertEqual(exc.args, (123,)) + cause = exc.__cause__ + self.assertIs(type(cause), futures.process._RemoteTraceback) + self.assertIn('raise RuntimeError(123) # some comment', cause.tb) + + with support.captured_stderr() as f1: + try: + raise exc + except RuntimeError: + sys.excepthook(*sys.exc_info()) + self.assertIn('raise RuntimeError(123) # some comment', + f1.getvalue()) + + @hashlib_helper.requires_hashdigest('md5') + def test_ressources_gced_in_workers(self): + # Ensure that argument for a job are correctly gc-ed after the job + # is finished + mgr = self.get_context().Manager() + obj = EventfulGCObj(mgr) + future = self.executor.submit(id, obj) + future.result() + + self.assertTrue(obj.event.wait(timeout=1)) + + # explicitly destroy the object to ensure that EventfulGCObj.__del__() + # is called while manager is still running. + obj = None + support.gc_collect() + + mgr.shutdown() + mgr.join() + + def test_saturation(self): + executor = self.executor + mp_context = self.get_context() + sem = mp_context.Semaphore(0) + job_count = 15 * executor._max_workers + for _ in range(job_count): + executor.submit(sem.acquire) + self.assertEqual(len(executor._processes), executor._max_workers) + for _ in range(job_count): + sem.release() + + def test_idle_process_reuse_one(self): + executor = self.executor + assert executor._max_workers >= 4 + if self.get_context().get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + executor.submit(mul, 21, 2).result() + executor.submit(mul, 6, 7).result() + executor.submit(mul, 3, 14).result() + self.assertEqual(len(executor._processes), 1) + + def test_idle_process_reuse_multiple(self): + executor = self.executor + assert executor._max_workers <= 5 + if self.get_context().get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + executor.submit(mul, 12, 7).result() + executor.submit(mul, 33, 25) + executor.submit(mul, 25, 26).result() + executor.submit(mul, 18, 29) + executor.submit(mul, 1, 2).result() + executor.submit(mul, 0, 9) + self.assertLessEqual(len(executor._processes), 3) + executor.shutdown() + + def test_max_tasks_per_child(self): + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + with self.assertRaises(ValueError): + self.executor_type(1, mp_context=context, max_tasks_per_child=3) + return + # not using self.executor as we need to control construction. + # arguably this could go in another class w/o that mixin. + executor = self.executor_type( + 1, mp_context=context, max_tasks_per_child=3) + f1 = executor.submit(os.getpid) + original_pid = f1.result() + # The worker pid remains the same as the worker could be reused + f2 = executor.submit(os.getpid) + self.assertEqual(f2.result(), original_pid) + self.assertEqual(len(executor._processes), 1) + f3 = executor.submit(os.getpid) + self.assertEqual(f3.result(), original_pid) + + # A new worker is spawned, with a statistically different pid, + # while the previous was reaped. + f4 = executor.submit(os.getpid) + new_pid = f4.result() + self.assertNotEqual(original_pid, new_pid) + self.assertEqual(len(executor._processes), 1) + + executor.shutdown() + + def test_max_tasks_per_child_defaults_to_spawn_context(self): + # not using self.executor as we need to control construction. + # arguably this could go in another class w/o that mixin. + executor = self.executor_type(1, max_tasks_per_child=3) + self.assertEqual(executor._mp_context.get_start_method(), "spawn") + + def test_max_tasks_early_shutdown(self): + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + # not using self.executor as we need to control construction. + # arguably this could go in another class w/o that mixin. + executor = self.executor_type( + 3, mp_context=context, max_tasks_per_child=1) + futures = [] + for i in range(6): + futures.append(executor.submit(mul, i, i)) + executor.shutdown() + for i, future in enumerate(futures): + self.assertEqual(future.result(), mul(i, i)) + + +create_executor_tests(globals(), ProcessPoolExecutorTest, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py new file mode 100644 index 00000000000000..45dab7a75fdd50 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -0,0 +1,343 @@ +import signal +import sys +import threading +import time +import unittest +from concurrent import futures + +from test import support +from test.support.script_helper import assert_python_ok + +from .util import ( + BaseTestCase, ThreadPoolMixin, ProcessPoolForkMixin, + ProcessPoolForkserverMixin, ProcessPoolSpawnMixin, + create_executor_tests, setup_module) + + +def sleep_and_print(t, msg): + time.sleep(t) + print(msg) + sys.stdout.flush() + + +class ExecutorShutdownTest: + def test_run_after_shutdown(self): + self.executor.shutdown() + self.assertRaises(RuntimeError, + self.executor.submit, + pow, 2, 5) + + def test_interpreter_shutdown(self): + # Test the atexit hook for shutdown of worker threads and processes + rc, out, err = assert_python_ok('-c', """if 1: + from concurrent.futures import {executor_type} + from time import sleep + from test.test_concurrent_futures.test_shutdown import sleep_and_print + if __name__ == "__main__": + context = '{context}' + if context == "": + t = {executor_type}(5) + else: + from multiprocessing import get_context + context = get_context(context) + t = {executor_type}(5, mp_context=context) + t.submit(sleep_and_print, 1.0, "apple") + """.format(executor_type=self.executor_type.__name__, + context=getattr(self, "ctx", ""))) + # Errors in atexit hooks don't change the process exit code, check + # stderr manually. + self.assertFalse(err) + self.assertEqual(out.strip(), b"apple") + + def test_submit_after_interpreter_shutdown(self): + # Test the atexit hook for shutdown of worker threads and processes + rc, out, err = assert_python_ok('-c', """if 1: + import atexit + @atexit.register + def run_last(): + try: + t.submit(id, None) + except RuntimeError: + print("runtime-error") + raise + from concurrent.futures import {executor_type} + if __name__ == "__main__": + context = '{context}' + if not context: + t = {executor_type}(5) + else: + from multiprocessing import get_context + context = get_context(context) + t = {executor_type}(5, mp_context=context) + t.submit(id, 42).result() + """.format(executor_type=self.executor_type.__name__, + context=getattr(self, "ctx", ""))) + # Errors in atexit hooks don't change the process exit code, check + # stderr manually. + self.assertIn("RuntimeError: cannot schedule new futures", err.decode()) + self.assertEqual(out.strip(), b"runtime-error") + + def test_hang_issue12364(self): + fs = [self.executor.submit(time.sleep, 0.1) for _ in range(50)] + self.executor.shutdown() + for f in fs: + f.result() + + def test_cancel_futures(self): + assert self.worker_count <= 5, "test needs few workers" + fs = [self.executor.submit(time.sleep, .1) for _ in range(50)] + self.executor.shutdown(cancel_futures=True) + # We can't guarantee the exact number of cancellations, but we can + # guarantee that *some* were cancelled. With few workers, many of + # the submitted futures should have been cancelled. + cancelled = [fut for fut in fs if fut.cancelled()] + self.assertGreater(len(cancelled), 20) + + # Ensure the other futures were able to finish. + # Use "not fut.cancelled()" instead of "fut.done()" to include futures + # that may have been left in a pending state. + others = [fut for fut in fs if not fut.cancelled()] + for fut in others: + self.assertTrue(fut.done(), msg=f"{fut._state=}") + self.assertIsNone(fut.exception()) + + # Similar to the number of cancelled futures, we can't guarantee the + # exact number that completed. But, we can guarantee that at least + # one finished. + self.assertGreater(len(others), 0) + + def test_hang_gh83386(self): + """shutdown(wait=False) doesn't hang at exit with running futures. + + See https://github.com/python/cpython/issues/83386. + """ + if self.executor_type == futures.ProcessPoolExecutor: + raise unittest.SkipTest( + "Hangs, see https://github.com/python/cpython/issues/83386") + + rc, out, err = assert_python_ok('-c', """if True: + from concurrent.futures import {executor_type} + from test.test_concurrent_futures.test_shutdown import sleep_and_print + if __name__ == "__main__": + if {context!r}: multiprocessing.set_start_method({context!r}) + t = {executor_type}(max_workers=3) + t.submit(sleep_and_print, 1.0, "apple") + t.shutdown(wait=False) + """.format(executor_type=self.executor_type.__name__, + context=getattr(self, 'ctx', None))) + self.assertFalse(err) + self.assertEqual(out.strip(), b"apple") + + def test_hang_gh94440(self): + """shutdown(wait=True) doesn't hang when a future was submitted and + quickly canceled right before shutdown. + + See https://github.com/python/cpython/issues/94440. + """ + if not hasattr(signal, 'alarm'): + raise unittest.SkipTest( + "Tested platform does not support the alarm signal") + + def timeout(_signum, _frame): + raise RuntimeError("timed out waiting for shutdown") + + kwargs = {} + if getattr(self, 'ctx', None): + kwargs['mp_context'] = self.get_context() + executor = self.executor_type(max_workers=1, **kwargs) + executor.submit(int).result() + old_handler = signal.signal(signal.SIGALRM, timeout) + try: + signal.alarm(5) + executor.submit(int).cancel() + executor.shutdown(wait=True) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + +class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase): + def test_threads_terminate(self): + def acquire_lock(lock): + lock.acquire() + + sem = threading.Semaphore(0) + for i in range(3): + self.executor.submit(acquire_lock, sem) + self.assertEqual(len(self.executor._threads), 3) + for i in range(3): + sem.release() + self.executor.shutdown() + for t in self.executor._threads: + t.join() + + def test_context_manager_shutdown(self): + with futures.ThreadPoolExecutor(max_workers=5) as e: + executor = e + self.assertEqual(list(e.map(abs, range(-5, 5))), + [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) + + for t in executor._threads: + t.join() + + def test_del_shutdown(self): + executor = futures.ThreadPoolExecutor(max_workers=5) + res = executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + + for t in threads: + t.join() + + # Make sure the results were all computed before the + # executor got shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + def test_shutdown_no_wait(self): + # Ensure that the executor cleans up the threads when calling + # shutdown with wait=False + executor = futures.ThreadPoolExecutor(max_workers=5) + res = executor.map(abs, range(-5, 5)) + threads = executor._threads + executor.shutdown(wait=False) + for t in threads: + t.join() + + # Make sure the results were all computed before the + # executor got shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + + def test_thread_names_assigned(self): + executor = futures.ThreadPoolExecutor( + max_workers=5, thread_name_prefix='SpecialPool') + executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + support.gc_collect() # For PyPy or other GCs. + + for t in threads: + self.assertRegex(t.name, r'^SpecialPool_[0-4]$') + t.join() + + def test_thread_names_default(self): + executor = futures.ThreadPoolExecutor(max_workers=5) + executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + support.gc_collect() # For PyPy or other GCs. + + for t in threads: + # Ensure that our default name is reasonably sane and unique when + # no thread_name_prefix was supplied. + self.assertRegex(t.name, r'ThreadPoolExecutor-\d+_[0-4]$') + t.join() + + def test_cancel_futures_wait_false(self): + # Can only be reliably tested for TPE, since PPE often hangs with + # `wait=False` (even without *cancel_futures*). + rc, out, err = assert_python_ok('-c', """if True: + from concurrent.futures import ThreadPoolExecutor + from test.test_concurrent_futures.test_shutdown import sleep_and_print + if __name__ == "__main__": + t = ThreadPoolExecutor() + t.submit(sleep_and_print, .1, "apple") + t.shutdown(wait=False, cancel_futures=True) + """) + # Errors in atexit hooks don't change the process exit code, check + # stderr manually. + self.assertFalse(err) + self.assertEqual(out.strip(), b"apple") + + +class ProcessPoolShutdownTest(ExecutorShutdownTest): + def test_processes_terminate(self): + def acquire_lock(lock): + lock.acquire() + + mp_context = self.get_context() + if mp_context.get_start_method(allow_none=False) == "fork": + # fork pre-spawns, not on demand. + expected_num_processes = self.worker_count + else: + expected_num_processes = 3 + + sem = mp_context.Semaphore(0) + for _ in range(3): + self.executor.submit(acquire_lock, sem) + self.assertEqual(len(self.executor._processes), expected_num_processes) + for _ in range(3): + sem.release() + processes = self.executor._processes + self.executor.shutdown() + + for p in processes.values(): + p.join() + + def test_context_manager_shutdown(self): + with futures.ProcessPoolExecutor( + max_workers=5, mp_context=self.get_context()) as e: + processes = e._processes + self.assertEqual(list(e.map(abs, range(-5, 5))), + [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) + + for p in processes.values(): + p.join() + + def test_del_shutdown(self): + executor = futures.ProcessPoolExecutor( + max_workers=5, mp_context=self.get_context()) + res = executor.map(abs, range(-5, 5)) + executor_manager_thread = executor._executor_manager_thread + processes = executor._processes + call_queue = executor._call_queue + executor_manager_thread = executor._executor_manager_thread + del executor + support.gc_collect() # For PyPy or other GCs. + + # Make sure that all the executor resources were properly cleaned by + # the shutdown process + executor_manager_thread.join() + for p in processes.values(): + p.join() + call_queue.join_thread() + + # Make sure the results were all computed before the + # executor got shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + def test_shutdown_no_wait(self): + # Ensure that the executor cleans up the processes when calling + # shutdown with wait=False + executor = futures.ProcessPoolExecutor( + max_workers=5, mp_context=self.get_context()) + res = executor.map(abs, range(-5, 5)) + processes = executor._processes + call_queue = executor._call_queue + executor_manager_thread = executor._executor_manager_thread + executor.shutdown(wait=False) + + # Make sure that all the executor resources were properly cleaned by + # the shutdown process + executor_manager_thread.join() + for p in processes.values(): + p.join() + call_queue.join_thread() + + # Make sure the results were all computed before the executor got + # shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + +create_executor_tests(globals(), ProcessPoolShutdownTest, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py new file mode 100644 index 00000000000000..daef7b5a836825 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -0,0 +1,98 @@ +import contextlib +import multiprocessing as mp +import multiprocessing.process +import multiprocessing.util +import os +import threading +import unittest +from concurrent import futures + +from .executor import ExecutorTest, mul +from .util import BaseTestCase, ThreadPoolMixin, setup_module + + +class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, BaseTestCase): + def test_map_submits_without_iteration(self): + """Tests verifying issue 11777.""" + finished = [] + def record_finished(n): + finished.append(n) + + self.executor.map(record_finished, range(10)) + self.executor.shutdown(wait=True) + self.assertCountEqual(finished, range(10)) + + def test_default_workers(self): + executor = self.executor_type() + expected = min(32, (os.cpu_count() or 1) + 4) + self.assertEqual(executor._max_workers, expected) + + def test_saturation(self): + executor = self.executor_type(4) + def acquire_lock(lock): + lock.acquire() + + sem = threading.Semaphore(0) + for i in range(15 * executor._max_workers): + executor.submit(acquire_lock, sem) + self.assertEqual(len(executor._threads), executor._max_workers) + for i in range(15 * executor._max_workers): + sem.release() + executor.shutdown(wait=True) + + def test_idle_thread_reuse(self): + executor = self.executor_type() + executor.submit(mul, 21, 2).result() + executor.submit(mul, 6, 7).result() + executor.submit(mul, 3, 14).result() + self.assertEqual(len(executor._threads), 1) + executor.shutdown(wait=True) + + @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + def test_hang_global_shutdown_lock(self): + # bpo-45021: _global_shutdown_lock should be reinitialized in the child + # process, otherwise it will never exit + def submit(pool): + pool.submit(submit, pool) + + with futures.ThreadPoolExecutor(1) as pool: + pool.submit(submit, pool) + + for _ in range(50): + with futures.ProcessPoolExecutor(1, mp_context=mp.get_context('fork')) as workers: + workers.submit(tuple) + + def test_executor_map_current_future_cancel(self): + stop_event = threading.Event() + log = [] + + def log_n_wait(ident): + log.append(f"{ident=} started") + try: + stop_event.wait() + finally: + log.append(f"{ident=} stopped") + + with self.executor_type(max_workers=1) as pool: + # submit work to saturate the pool + fut = pool.submit(log_n_wait, ident="first") + try: + with contextlib.closing( + pool.map(log_n_wait, ["second", "third"], timeout=0) + ) as gen: + with self.assertRaises(TimeoutError): + next(gen) + finally: + stop_event.set() + fut.result() + # ident='second' is cancelled as a result of raising a TimeoutError + # ident='third' is cancelled because it remained in the collection of futures + self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"]) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py new file mode 100644 index 00000000000000..e4bea8b05aced6 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -0,0 +1,161 @@ +import sys +import threading +import time +import unittest +from concurrent import futures + +from .util import ( + CANCELLED_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + create_executor_tests, setup_module, + BaseTestCase, ThreadPoolMixin, + ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin) + + +def mul(x, y): + return x * y + +def sleep_and_raise(t): + time.sleep(t) + raise Exception('this is an exception') + + +class WaitTests: + def test_20369(self): + # See https://bugs.python.org/issue20369 + future = self.executor.submit(time.sleep, 1.5) + done, not_done = futures.wait([future, future], + return_when=futures.ALL_COMPLETED) + self.assertEqual({future}, done) + self.assertEqual(set(), not_done) + + + def test_first_completed(self): + future1 = self.executor.submit(mul, 21, 2) + future2 = self.executor.submit(time.sleep, 1.5) + + done, not_done = futures.wait( + [CANCELLED_FUTURE, future1, future2], + return_when=futures.FIRST_COMPLETED) + + self.assertEqual(set([future1]), done) + self.assertEqual(set([CANCELLED_FUTURE, future2]), not_done) + + def test_first_completed_some_already_completed(self): + future1 = self.executor.submit(time.sleep, 1.5) + + finished, pending = futures.wait( + [CANCELLED_AND_NOTIFIED_FUTURE, SUCCESSFUL_FUTURE, future1], + return_when=futures.FIRST_COMPLETED) + + self.assertEqual( + set([CANCELLED_AND_NOTIFIED_FUTURE, SUCCESSFUL_FUTURE]), + finished) + self.assertEqual(set([future1]), pending) + + def test_first_exception(self): + future1 = self.executor.submit(mul, 2, 21) + future2 = self.executor.submit(sleep_and_raise, 1.5) + future3 = self.executor.submit(time.sleep, 3) + + finished, pending = futures.wait( + [future1, future2, future3], + return_when=futures.FIRST_EXCEPTION) + + self.assertEqual(set([future1, future2]), finished) + self.assertEqual(set([future3]), pending) + + def test_first_exception_some_already_complete(self): + future1 = self.executor.submit(divmod, 21, 0) + future2 = self.executor.submit(time.sleep, 1.5) + + finished, pending = futures.wait( + [SUCCESSFUL_FUTURE, + CANCELLED_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + future1, future2], + return_when=futures.FIRST_EXCEPTION) + + self.assertEqual(set([SUCCESSFUL_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + future1]), finished) + self.assertEqual(set([CANCELLED_FUTURE, future2]), pending) + + def test_first_exception_one_already_failed(self): + future1 = self.executor.submit(time.sleep, 2) + + finished, pending = futures.wait( + [EXCEPTION_FUTURE, future1], + return_when=futures.FIRST_EXCEPTION) + + self.assertEqual(set([EXCEPTION_FUTURE]), finished) + self.assertEqual(set([future1]), pending) + + def test_all_completed(self): + future1 = self.executor.submit(divmod, 2, 0) + future2 = self.executor.submit(mul, 2, 21) + + finished, pending = futures.wait( + [SUCCESSFUL_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + future1, + future2], + return_when=futures.ALL_COMPLETED) + + self.assertEqual(set([SUCCESSFUL_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + future1, + future2]), finished) + self.assertEqual(set(), pending) + + def test_timeout(self): + future1 = self.executor.submit(mul, 6, 7) + future2 = self.executor.submit(time.sleep, 6) + + finished, pending = futures.wait( + [CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future1, future2], + timeout=5, + return_when=futures.ALL_COMPLETED) + + self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future1]), finished) + self.assertEqual(set([future2]), pending) + + +class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): + + def test_pending_calls_race(self): + # Issue #14406: multi-threaded race condition when waiting on all + # futures. + event = threading.Event() + def future_func(): + event.wait() + oldswitchinterval = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + fs = {self.executor.submit(future_func) for i in range(100)} + event.set() + futures.wait(fs, return_when=futures.ALL_COMPLETED) + finally: + sys.setswitchinterval(oldswitchinterval) + + +create_executor_tests(globals(), WaitTests, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py new file mode 100644 index 00000000000000..dc48bec796b87f --- /dev/null +++ b/Lib/test/test_concurrent_futures/util.py @@ -0,0 +1,141 @@ +import multiprocessing +import sys +import time +import unittest +from concurrent import futures +from concurrent.futures._base import ( + PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future, + ) +from concurrent.futures.process import _check_system_limits + +from test import support +from test.support import threading_helper + + +def create_future(state=PENDING, exception=None, result=None): + f = Future() + f._state = state + f._exception = exception + f._result = result + return f + + +PENDING_FUTURE = create_future(state=PENDING) +RUNNING_FUTURE = create_future(state=RUNNING) +CANCELLED_FUTURE = create_future(state=CANCELLED) +CANCELLED_AND_NOTIFIED_FUTURE = create_future(state=CANCELLED_AND_NOTIFIED) +EXCEPTION_FUTURE = create_future(state=FINISHED, exception=OSError()) +SUCCESSFUL_FUTURE = create_future(state=FINISHED, result=42) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._thread_key = threading_helper.threading_setup() + + def tearDown(self): + support.reap_children() + threading_helper.threading_cleanup(*self._thread_key) + + +class ExecutorMixin: + worker_count = 5 + executor_kwargs = {} + + def setUp(self): + super().setUp() + + self.t1 = time.monotonic() + if hasattr(self, "ctx"): + self.executor = self.executor_type( + max_workers=self.worker_count, + mp_context=self.get_context(), + **self.executor_kwargs) + else: + self.executor = self.executor_type( + max_workers=self.worker_count, + **self.executor_kwargs) + + def tearDown(self): + self.executor.shutdown(wait=True) + self.executor = None + + dt = time.monotonic() - self.t1 + if support.verbose: + print("%.2fs" % dt, end=' ') + self.assertLess(dt, 300, "synchronization issue: test lasted too long") + + super().tearDown() + + def get_context(self): + return multiprocessing.get_context(self.ctx) + + +class ThreadPoolMixin(ExecutorMixin): + executor_type = futures.ThreadPoolExecutor + + +class ProcessPoolForkMixin(ExecutorMixin): + executor_type = futures.ProcessPoolExecutor + ctx = "fork" + + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + if sys.platform == "win32": + self.skipTest("require unix system") + return super().get_context() + + +class ProcessPoolSpawnMixin(ExecutorMixin): + executor_type = futures.ProcessPoolExecutor + ctx = "spawn" + + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + return super().get_context() + + +class ProcessPoolForkserverMixin(ExecutorMixin): + executor_type = futures.ProcessPoolExecutor + ctx = "forkserver" + + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + if sys.platform == "win32": + self.skipTest("require unix system") + return super().get_context() + + +def create_executor_tests(remote_globals, mixin, bases=(BaseTestCase,), + executor_mixins=(ThreadPoolMixin, + ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)): + def strip_mixin(name): + if name.endswith(('Mixin', 'Tests')): + return name[:-5] + elif name.endswith('Test'): + return name[:-4] + else: + return name + + module = remote_globals['__name__'] + for exe in executor_mixins: + name = ("%s%sTest" + % (strip_mixin(exe.__name__), strip_mixin(mixin.__name__))) + cls = type(name, (mixin,) + (exe,) + bases, {'__module__': module}) + remote_globals[name] = cls + + +def setup_module(): + unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Misc/NEWS.d/next/Tests/2023-08-24-06-10-36.gh-issue-108388.YCVB0D.rst b/Misc/NEWS.d/next/Tests/2023-08-24-06-10-36.gh-issue-108388.YCVB0D.rst new file mode 100644 index 00000000000000..ddff07be024d47 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-24-06-10-36.gh-issue-108388.YCVB0D.rst @@ -0,0 +1,2 @@ +Convert test_concurrent_futures to a package of 7 sub-tests. Patch by Victor +Stinner. From 480a337366f4a5335c599684609d5f59f27805b9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 19:37:41 +0200 Subject: [PATCH 305/598] gh-106320: Remove private _PyContext_NewHamtForTests() (#108434) Move the function to the internal C API. --- Include/cpython/context.h | 4 ---- Include/internal/pycore_context.h | 5 +++++ Lib/test/test_context.py | 4 ++-- Modules/_testcapimodule.c | 8 -------- Modules/_testinternalcapi.c | 11 ++++++++++- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/Include/cpython/context.h b/Include/cpython/context.h index 9879fc7192ebb8..a3249fc29b082e 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -67,10 +67,6 @@ PyAPI_FUNC(PyObject *) PyContextVar_Set(PyObject *var, PyObject *value); PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token); -/* This method is exposed only for CPython tests. Don not use it. */ -PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void); - - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index f1898cf873312b..ec884e9e0f55a9 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -68,4 +68,9 @@ struct _pycontexttokenobject { }; +// _testinternalcapi.hamt() used by tests. +// Export for '_testcapi' shared extension +PyAPI_FUNC(PyObject*) _PyContext_NewHamtForTests(void); + + #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index b1aece4f5c9c49..b72d5addc6bd7d 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -9,7 +9,7 @@ from test.support import threading_helper try: - from _testcapi import hamt + from _testinternalcapi import hamt except ImportError: hamt = None @@ -431,7 +431,7 @@ class EqError(Exception): pass -@unittest.skipIf(hamt is None, '_testcapi lacks "hamt()" function') +@unittest.skipIf(hamt is None, '_testinternalcapi.hamt() not available') class HamtTest(unittest.TestCase): def test_hashkey_helper_1(self): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a7a98d1eea5bd1..7086d321d4aaa2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2125,13 +2125,6 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) } -static PyObject* -new_hamt(PyObject *self, PyObject *args) -{ - return _PyContext_NewHamtForTests(); -} - - /* def bad_get(self, obj, cls): cls() return repr(self) @@ -3626,7 +3619,6 @@ static PyMethodDef TestMethods[] = { {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, - {"hamt", new_hamt, METH_NOARGS}, {"bad_get", _PyCFunction_CAST(bad_get), METH_FASTCALL}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 7b98885189f5c0..9b45d59987dca6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -13,8 +13,9 @@ #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc #include "pycore_ceval.h" // _PyEval_AddPendingCall() +#include "pycore_compile.h" // _PyCompile_CodeGen() +#include "pycore_context.h" // _PyContext_NewHamtForTests() #include "pycore_dict.h" // _PyDictOrValues_GetValues() #include "pycore_fileutils.h" // _Py_normpath() #include "pycore_frame.h" // _PyInterpreterFrame @@ -1564,6 +1565,13 @@ get_object_dict_values(PyObject *self, PyObject *obj) } +static PyObject* +new_hamt(PyObject *self, PyObject *args) +{ + return _PyContext_NewHamtForTests(); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1628,6 +1636,7 @@ static PyMethodDef module_functions[] = { check_pyobject_uninitialized_is_freed, METH_NOARGS}, {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, {"get_object_dict_values", get_object_dict_values, METH_O}, + {"hamt", new_hamt, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; From fa626b8ca019c36418a1ddcefcf9ba1f52946609 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 19:43:29 +0200 Subject: [PATCH 306/598] gh-107178: Remove _testcapi.test_dict_capi() (#108436) The new _testcapi.test_dict tests are more complete, _testcapi.test_dict_capi() became redundant. --- Modules/_testcapimodule.c | 191 -------------------------------------- 1 file changed, 191 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 7086d321d4aaa2..6a1eba3a0de217 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3318,196 +3318,6 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } -static PyObject * -test_dict_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) -{ - assert(!PyErr_Occurred()); - - PyObject *dict= NULL, *key = NULL, *missing_key = NULL, *value = NULL; - PyObject *invalid_key = NULL; - int res; - - // test PyDict_New() - dict = PyDict_New(); - if (dict == NULL) { - goto error; - } - - key = PyUnicode_FromString("key"); - if (key == NULL) { - goto error; - } - - missing_key = PyUnicode_FromString("missing_key"); - if (missing_key == NULL) { - goto error; - } - - value = PyUnicode_FromString("value"); - if (value == NULL) { - goto error; - } - - // test PyDict_SetItem() - Py_ssize_t key_refcnt = Py_REFCNT(key); - Py_ssize_t value_refcnt = Py_REFCNT(value); - res = PyDict_SetItem(dict, key, value); - if (res < 0) { - goto error; - } - assert(res == 0); - assert(Py_REFCNT(key) == (key_refcnt + 1)); - assert(Py_REFCNT(value) == (value_refcnt + 1)); - - // test PyDict_SetItemString() - res = PyDict_SetItemString(dict, "key", value); - if (res < 0) { - goto error; - } - assert(res == 0); - assert(Py_REFCNT(key) == (key_refcnt + 1)); - assert(Py_REFCNT(value) == (value_refcnt + 1)); - - // test PyDict_Size() - assert(PyDict_Size(dict) == 1); - - // test PyDict_Contains(), key is present - assert(PyDict_Contains(dict, key) == 1); - - // test PyDict_GetItem(), key is present - assert(PyDict_GetItem(dict, key) == value); - - // test PyDict_GetItemString(), key is present - assert(PyDict_GetItemString(dict, "key") == value); - - // test PyDict_GetItemWithError(), key is present - assert(PyDict_GetItemWithError(dict, key) == value); - assert(!PyErr_Occurred()); - - // test PyDict_GetItemRef(), key is present - PyObject *get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemRef(dict, key, &get_value) == 1); - assert(get_value == value); - Py_DECREF(get_value); - - // test PyDict_GetItemStringRef(), key is present - get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemStringRef(dict, "key", &get_value) == 1); - assert(get_value == value); - Py_DECREF(get_value); - - // test PyDict_Contains(), missing key - assert(PyDict_Contains(dict, missing_key) == 0); - - // test PyDict_GetItem(), missing key - assert(PyDict_GetItem(dict, missing_key) == NULL); - assert(!PyErr_Occurred()); - - // test PyDict_GetItemString(), missing key - assert(PyDict_GetItemString(dict, "missing_key") == NULL); - assert(!PyErr_Occurred()); - - // test PyDict_GetItemWithError(), missing key - assert(PyDict_GetItem(dict, missing_key) == NULL); - assert(!PyErr_Occurred()); - - // test PyDict_GetItemRef(), missing key - get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemRef(dict, missing_key, &get_value) == 0); - assert(!PyErr_Occurred()); - assert(get_value == NULL); - - // test PyDict_GetItemStringRef(), missing key - get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemStringRef(dict, "missing_key", &get_value) == 0); - assert(!PyErr_Occurred()); - assert(get_value == NULL); - - // test PyDict_GetItem(), invalid dict - PyObject *invalid_dict = key; // borrowed reference - assert(PyDict_GetItem(invalid_dict, key) == NULL); - assert(!PyErr_Occurred()); - - // test PyDict_GetItemWithError(), invalid dict - assert(PyDict_GetItemWithError(invalid_dict, key) == NULL); - assert(PyErr_ExceptionMatches(PyExc_SystemError)); - PyErr_Clear(); - - // test PyDict_GetItemRef(), invalid dict - get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemRef(invalid_dict, key, &get_value) == -1); - assert(PyErr_ExceptionMatches(PyExc_SystemError)); - PyErr_Clear(); - assert(get_value == NULL); - - // test PyDict_GetItemStringRef(), invalid dict - get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemStringRef(invalid_dict, "key", &get_value) == -1); - assert(PyErr_ExceptionMatches(PyExc_SystemError)); - PyErr_Clear(); - assert(get_value == NULL); - - invalid_key = PyList_New(0); - if (invalid_key == NULL) { - goto error; - } - - // test PyDict_Contains(), invalid key - assert(PyDict_Contains(dict, invalid_key) == -1); - assert(PyErr_ExceptionMatches(PyExc_TypeError)); - PyErr_Clear(); - - // test PyDict_GetItem(), invalid key - assert(PyDict_GetItem(dict, invalid_key) == NULL); - assert(!PyErr_Occurred()); - - // test PyDict_GetItemWithError(), invalid key - assert(PyDict_GetItemWithError(dict, invalid_key) == NULL); - assert(PyErr_ExceptionMatches(PyExc_TypeError)); - PyErr_Clear(); - - // test PyDict_GetItemRef(), invalid key - get_value = Py_Ellipsis; // marker value - assert(PyDict_GetItemRef(dict, invalid_key, &get_value) == -1); - assert(PyErr_ExceptionMatches(PyExc_TypeError)); - PyErr_Clear(); - assert(get_value == NULL); - - // test PyDict_DelItem(), key is present - assert(PyDict_DelItem(dict, key) == 0); - assert(PyDict_Size(dict) == 0); - - // test PyDict_DelItem(), missing key - assert(PyDict_DelItem(dict, missing_key) == -1); - assert(PyErr_ExceptionMatches(PyExc_KeyError)); - PyErr_Clear(); - - // test PyDict_DelItem(), invalid key - assert(PyDict_DelItem(dict, invalid_key) == -1); - assert(PyErr_ExceptionMatches(PyExc_TypeError)); - PyErr_Clear(); - - // test PyDict_Clear() - PyDict_Clear(dict); - - Py_DECREF(dict); - Py_DECREF(key); - Py_DECREF(missing_key); - Py_DECREF(value); - Py_DECREF(invalid_key); - - Py_RETURN_NONE; - -error: - Py_XDECREF(dict); - Py_XDECREF(key); - Py_XDECREF(missing_key); - Py_XDECREF(value); - Py_XDECREF(invalid_key); - return NULL; -} - - static PyObject * sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -3670,7 +3480,6 @@ static PyMethodDef TestMethods[] = { {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, - {"test_dict_capi", test_dict_capi, METH_NOARGS}, {"sys_getobject", sys_getobject, METH_O}, {"sys_setobject", sys_setobject, METH_VARARGS}, {NULL, NULL} /* sentinel */ From fa6933e035ba81ae3ed6ceac61b901a3d52b1e30 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 19:47:44 +0200 Subject: [PATCH 307/598] gh-107211: Fix test_peg_generator (#108435) Export these internal functions for test_peg_generator * _PyArena_AddPyObject() * _PyArena_Free() * _PyArena_Malloc() * _PyArena_New() --- Include/internal/pycore_pyarena.h | 92 ++++++++++++++++--------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/Include/internal/pycore_pyarena.h b/Include/internal/pycore_pyarena.h index 6bd926a95cda83..1f07479fb2ca27 100644 --- a/Include/internal/pycore_pyarena.h +++ b/Include/internal/pycore_pyarena.h @@ -1,7 +1,4 @@ -/* An arena-like memory interface for the compiler. - * - * Export symbols for test_peg_generator. - */ +// An arena-like memory interface for the compiler. #ifndef Py_INTERNAL_PYARENA_H #define Py_INTERNAL_PYARENA_H @@ -15,50 +12,55 @@ extern "C" { typedef struct _arena PyArena; -/* _PyArena_New() and _PyArena_Free() create a new arena and free it, - respectively. Once an arena has been created, it can be used - to allocate memory via _PyArena_Malloc(). Pointers to PyObject can - also be registered with the arena via _PyArena_AddPyObject(), and the - arena will ensure that the PyObjects stay alive at least until - _PyArena_Free() is called. When an arena is freed, all the memory it - allocated is freed, the arena releases internal references to registered - PyObject*, and none of its pointers are valid. - XXX (tim) What does "none of its pointers are valid" mean? Does it - XXX mean that pointers previously obtained via _PyArena_Malloc() are - XXX no longer valid? (That's clearly true, but not sure that's what - XXX the text is trying to say.) +// _PyArena_New() and _PyArena_Free() create a new arena and free it, +// respectively. Once an arena has been created, it can be used +// to allocate memory via _PyArena_Malloc(). Pointers to PyObject can +// also be registered with the arena via _PyArena_AddPyObject(), and the +// arena will ensure that the PyObjects stay alive at least until +// _PyArena_Free() is called. When an arena is freed, all the memory it +// allocated is freed, the arena releases internal references to registered +// PyObject*, and none of its pointers are valid. +// XXX (tim) What does "none of its pointers are valid" mean? Does it +// XXX mean that pointers previously obtained via _PyArena_Malloc() are +// XXX no longer valid? (That's clearly true, but not sure that's what +// XXX the text is trying to say.) +// +// _PyArena_New() returns an arena pointer. On error, it +// returns a negative number and sets an exception. +// XXX (tim): Not true. On error, _PyArena_New() actually returns NULL, +// XXX and looks like it may or may not set an exception (e.g., if the +// XXX internal PyList_New(0) returns NULL, _PyArena_New() passes that on +// XXX and an exception is set; OTOH, if the internal +// XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but +// XXX an exception is not set in that case). +// +// Export for test_peg_generator +PyAPI_FUNC(PyArena*) _PyArena_New(void); - _PyArena_New() returns an arena pointer. On error, it - returns a negative number and sets an exception. - XXX (tim): Not true. On error, _PyArena_New() actually returns NULL, - XXX and looks like it may or may not set an exception (e.g., if the - XXX internal PyList_New(0) returns NULL, _PyArena_New() passes that on - XXX and an exception is set; OTOH, if the internal - XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but - XXX an exception is not set in that case). -*/ -extern PyArena* _PyArena_New(void); -extern void _PyArena_Free(PyArena *); +// Export for test_peg_generator +PyAPI_FUNC(void) _PyArena_Free(PyArena *); -/* Mostly like malloc(), return the address of a block of memory spanning - * `size` bytes, or return NULL (without setting an exception) if enough - * new memory can't be obtained. Unlike malloc(0), _PyArena_Malloc() with - * size=0 does not guarantee to return a unique pointer (the pointer - * returned may equal one or more other pointers obtained from - * _PyArena_Malloc()). - * Note that pointers obtained via _PyArena_Malloc() must never be passed to - * the system free() or realloc(), or to any of Python's similar memory- - * management functions. _PyArena_Malloc()-obtained pointers remain valid - * until _PyArena_Free(ar) is called, at which point all pointers obtained - * from the arena `ar` become invalid simultaneously. - */ -extern void* _PyArena_Malloc(PyArena *, size_t size); +// Mostly like malloc(), return the address of a block of memory spanning +// `size` bytes, or return NULL (without setting an exception) if enough +// new memory can't be obtained. Unlike malloc(0), _PyArena_Malloc() with +// size=0 does not guarantee to return a unique pointer (the pointer +// returned may equal one or more other pointers obtained from +// _PyArena_Malloc()). +// Note that pointers obtained via _PyArena_Malloc() must never be passed to +// the system free() or realloc(), or to any of Python's similar memory- +// management functions. _PyArena_Malloc()-obtained pointers remain valid +// until _PyArena_Free(ar) is called, at which point all pointers obtained +// from the arena `ar` become invalid simultaneously. +// +// Export for test_peg_generator +PyAPI_FUNC(void*) _PyArena_Malloc(PyArena *, size_t size); -/* This routine isn't a proper arena allocation routine. It takes - * a PyObject* and records it so that it can be DECREFed when the - * arena is freed. - */ -extern int _PyArena_AddPyObject(PyArena *, PyObject *); +// This routine isn't a proper arena allocation routine. It takes +// a PyObject* and records it so that it can be DECREFed when the +// arena is freed. +// +// Export for test_peg_generator +PyAPI_FUNC(int) _PyArena_AddPyObject(PyArena *, PyObject *); #ifdef __cplusplus } From 995f4c48e11349fbfb9233e02b732d4534d3008e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 20:09:23 +0200 Subject: [PATCH 308/598] gh-80527: Change support.requires_legacy_unicode_capi() (#108438) The decorator now requires to be called with parenthesis: @support.requires_legacy_unicode_capi() instead of: @support.requires_legacy_unicode_capi The implementation now only imports _testcapi when the decorator is called, so "import test.support" no longer imports the _testcapi extension. --- Lib/test/support/__init__.py | 15 ++++++++------- Lib/test/test_capi/test_getargs.py | 8 ++++---- Lib/test/test_csv.py | 2 +- Lib/test/test_decimal.py | 4 ++-- Lib/test/test_str.py | 4 ++-- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8e8bf49a5cb5ce..328bddbdc6887b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -21,11 +21,6 @@ from .testresult import get_test_runner -try: - from _testcapi import unicode_legacy_string -except ImportError: - unicode_legacy_string = None - __all__ = [ # globals "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast", @@ -507,8 +502,14 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) -requires_legacy_unicode_capi = unittest.skipUnless(unicode_legacy_string, - 'requires legacy Unicode C API') +def requires_legacy_unicode_capi(): + try: + from _testcapi import unicode_legacy_string + except ImportError: + unicode_legacy_string = None + + return unittest.skipUnless(unicode_legacy_string, + 'requires legacy Unicode C API') # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 01bc30779add73..246206af86101c 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1004,7 +1004,7 @@ def test_et_hash(self): buf = bytearray() self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() def test_u(self): from _testcapi import getargs_u with self.assertWarns(DeprecationWarning): @@ -1020,7 +1020,7 @@ def test_u(self): with self.assertWarns(DeprecationWarning): self.assertRaises(TypeError, getargs_u, None) - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() def test_u_hash(self): from _testcapi import getargs_u_hash with self.assertWarns(DeprecationWarning): @@ -1036,7 +1036,7 @@ def test_u_hash(self): with self.assertWarns(DeprecationWarning): self.assertRaises(TypeError, getargs_u_hash, None) - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() def test_Z(self): from _testcapi import getargs_Z with self.assertWarns(DeprecationWarning): @@ -1052,7 +1052,7 @@ def test_Z(self): with self.assertWarns(DeprecationWarning): self.assertIsNone(getargs_Z(None)) - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() def test_Z_hash(self): from _testcapi import getargs_Z_hash with self.assertWarns(DeprecationWarning): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 6a4180e6d1b0a1..bc6879176cd85e 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -282,7 +282,7 @@ def test_writerows_errors(self): self.assertRaises(OSError, writer.writerows, BadIterable()) @support.cpython_only - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_writerows_legacy_strings(self): import _testcapi diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index fc66a309788ac1..abfd71c868d009 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -588,7 +588,7 @@ def test_explicit_from_string(self): self.assertRaises(InvalidOperation, Decimal, "1_2_\u00003") @cpython_only - @requires_legacy_unicode_capi + @requires_legacy_unicode_capi() @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_from_legacy_strings(self): import _testcapi @@ -2920,7 +2920,7 @@ def test_none_args(self): Overflow]) @cpython_only - @requires_legacy_unicode_capi + @requires_legacy_unicode_capi() @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_from_legacy_strings(self): import _testcapi diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index a6bcc2455de15e..3ae2f45ef6bddc 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -813,7 +813,7 @@ def test_isidentifier(self): self.assertFalse("0".isidentifier()) @support.cpython_only - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_isidentifier_legacy(self): u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊' @@ -2490,7 +2490,7 @@ def test_getnewargs(self): self.assertEqual(len(args), 1) @support.cpython_only - @support.requires_legacy_unicode_capi + @support.requires_legacy_unicode_capi() @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_resize(self): for length in range(1, 100, 7): From c494fb333b57bdf43fc90189fc29a00c293b2987 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 20:25:22 +0200 Subject: [PATCH 309/598] gh-106320: Remove private _PyEval function (#108433) Move private _PyEval functions to the internal C API (pycore_ceval.h): * _PyEval_GetBuiltin() * _PyEval_GetBuiltinId() * _PyEval_GetSwitchInterval() * _PyEval_MakePendingCalls() * _PyEval_SetProfile() * _PyEval_SetSwitchInterval() * _PyEval_SetTrace() No longer export most of these functions. --- Include/cpython/ceval.h | 10 ---------- Include/internal/pycore_ceval.h | 17 +++++++++++++++++ Modules/_lsprof.c | 1 + Modules/_queuemodule.c | 1 + Modules/_threadmodule.c | 1 + Modules/arraymodule.c | 1 + Modules/itertoolsmodule.c | 1 + Objects/bytearrayobject.c | 1 + Objects/bytesobject.c | 5 +++-- Objects/classobject.c | 1 + Objects/dictobject.c | 1 + Objects/genericaliasobject.c | 1 + Objects/iterobject.c | 1 + Objects/listobject.c | 1 + Objects/odictobject.c | 1 + Objects/rangeobject.c | 1 + Objects/setobject.c | 1 + Objects/tupleobject.c | 1 + Objects/unicodeobject.c | 2 ++ 19 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 5255d715142b97..78f7405661662f 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -4,14 +4,9 @@ PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetProfileAllThreads(Py_tracefunc, PyObject *); -PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetTraceAllThreads(Py_tracefunc, PyObject *); -PyAPI_FUNC(int) _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); -/* Helper to look up a builtin object */ -PyAPI_FUNC(PyObject *) _PyEval_GetBuiltin(PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *); /* Look at the current frame's (if any) code's co_flags, and turn on the corresponding compiler flags in cf->cf_flags. Return 1 if any flag was set, else return 0. */ @@ -19,11 +14,6 @@ PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf); PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc); -PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); -PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); - -PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); - PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc); // Old name -- remove when this API changes: _Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index db3dac48662326..e9535023cec46b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -15,6 +15,23 @@ extern "C" { struct pyruntimestate; struct _ceval_runtime_state; +// Export for '_lsprof' shared extension +PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); + +extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); + +// Helper to look up a builtin object +// Export for 'array' shared extension +PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); + +extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); + +extern void _PyEval_SetSwitchInterval(unsigned long microseconds); +extern unsigned long _PyEval_GetSwitchInterval(void); + +// Export for '_queue' shared extension +PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); + #ifndef Py_DEFAULT_RECURSION_LIMIT # define Py_DEFAULT_RECURSION_LIMIT 1000 #endif diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 257de4387c0ab9..e7dcb6e1713212 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() #include "rotatingtree.h" diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index b0a36f03694507..b4bafb375c999d 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -3,6 +3,7 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _PyEval_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_time.h" // _PyTime_t diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 52f44d04523459..984747c44d7a57 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -3,6 +3,7 @@ /* Interface to Sjoerd's portable C thread library */ #include "Python.h" +#include "pycore_ceval.h" // _PyEval_MakePendingCalls() #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 5c1e6cb9cffbc3..309a36919f3465 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_long.h" // _PyLong_FromByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 0ab6d330e87793..4ed34aa5bde827 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_typeobject.h" // _PyType_GetModuleState() diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 18a24a369a64c1..67073190cc889d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -4,6 +4,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" #include "pycore_bytesobject.h" +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_strhex.h" // _Py_strhex_with_sep() #include "pycore_long.h" // _PyLong_FromUnsignedChar() diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index afe9192720c628..a2c1f4cf359f91 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2,11 +2,12 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_bytesobject.h" // _PyBytes_Find(), _PyBytes_Repeat() #include "pycore_bytes_methods.h" // _Py_bytes_startswith() +#include "pycore_bytesobject.h" // _PyBytes_Find(), _PyBytes_Repeat() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_format.h" // F_LJUST -#include "pycore_global_objects.h" // _Py_GET_GLOBAL_OBJECT() +#include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue #include "pycore_object.h" // _PyObject_GC_TRACK diff --git a/Objects/classobject.c b/Objects/classobject.c index 5471045d777c9d..618d88894debbe 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_VectorcallTstate() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d98e9a395c7eae..7cbc49f96994f7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -115,6 +115,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "Python.h" #include "pycore_bitutils.h" // _Py_bit_length #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_code.h" // stats #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index faf517b66b9350..bb50537461040b 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -1,6 +1,7 @@ // types.GenericAlias -- used to represent e.g. list[int]. #include "Python.h" +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_object.h" #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check diff --git a/Objects/iterobject.c b/Objects/iterobject.c index cf7cb8af52a274..135ced9ea1f268 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyObject_HasLen() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_object.h" // _PyObject_GC_TRACK() typedef struct { diff --git a/Objects/listobject.c b/Objects/listobject.c index c0da9dd916851a..ad1840b01226ec 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_state, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount diff --git a/Objects/odictobject.c b/Objects/odictobject.c index e76d2ded61cf74..d7a0f914d41aff 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -465,6 +465,7 @@ Potential Optimizations */ #include "Python.h" +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_dict.h" // _Py_dict_lookup() diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 6e06bef95032cf..ce9eef69ad75a8 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_range.h" diff --git a/Objects/setobject.c b/Objects/setobject.c index c96b62e38ec27e..14b53c12dda57f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -32,6 +32,7 @@ */ #include "Python.h" +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index b669a3dd8525eb..d567839c5e3a0b 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index c6876d4ca0ef0f..6bbc2af258e675 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -43,6 +43,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_atomic_funcs.h" // _Py_atomic_size_get() #include "pycore_bytes_methods.h" // _Py_bytes_lower() #include "pycore_bytesobject.h" // _PyBytes_Repeat() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_codecs.h" // _PyCodec_Lookup() #include "pycore_format.h" // F_LJUST #include "pycore_initconfig.h" // _PyStatus_OK() @@ -56,6 +57,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject_generated.h" // _PyUnicode_InitStaticStrings() + #include "stringlib/eq.h" // unicode_eq() #include // ptrdiff_t From 88941d665fc6b345f38b9147a7321e40019964d5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 24 Aug 2023 12:10:51 -0700 Subject: [PATCH 310/598] gh-106581: Fix two bugs in the code generator's copy optimization (#108380) I was comparing the last preceding poke with the *last* peek, rather than the *first* peek. Unfortunately this bug obscured another bug: When the last preceding poke is UNUSED, the first peek disappears, leaving the variable unassigned. This is how I fixed it: - Rename CopyEffect to CopyItem. - Change CopyItem to contain StackItems instead of StackEffects. - Update those StackItems when adjusting the manager higher or lower. - Assert that those StackItems' offsets are equivalent. - Other clever things. --------- Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Python/generated_cases.c.h | 3 -- Tools/cases_generator/stacking.py | 79 ++++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f6322df566c650..1724d11231727f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3833,7 +3833,6 @@ DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE - callable = stack_pointer[-2 - oparg]; { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -3841,8 +3840,6 @@ } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 1e117f11825938..2d006ba8e5934a 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -83,6 +83,27 @@ def as_index(self) -> str: terms = self.as_terms() return make_index(terms) + def equivalent_to(self, other: "StackOffset") -> bool: + if self.deep == other.deep and self.high == other.high: + return True + deep = list(self.deep) + for x in other.deep: + try: + deep.remove(x) + except ValueError: + return False + if deep: + return False + high = list(self.high) + for x in other.high: + try: + high.remove(x) + except ValueError: + return False + if high: + return False + return True + def make_index(terms: list[tuple[str, str]]) -> str: # Produce an index expression from the terms honoring PEP 8, @@ -131,9 +152,9 @@ def as_stack_effect(self, lax: bool = False) -> StackEffect: @dataclasses.dataclass -class CopyEffect: - src: StackEffect - dst: StackEffect +class CopyItem: + src: StackItem + dst: StackItem class EffectManager: @@ -143,7 +164,7 @@ class EffectManager: active_caches: list[ActiveCacheEffect] peeks: list[StackItem] pokes: list[StackItem] - copies: list[CopyEffect] # See merge() + copies: list[CopyItem] # See merge() # Track offsets from stack pointer min_offset: StackOffset final_offset: StackOffset @@ -179,16 +200,18 @@ def __init__( while ( pred.pokes and self.peeks - and pred.pokes[-1].effect == self.peeks[-1].effect + and pred.pokes[-1].effect == self.peeks[0].effect ): - src = pred.pokes.pop(-1).effect - dst = self.peeks.pop(0).effect - pred.final_offset.deeper(src) - if dst.name != UNUSED: - destinations.add(dst.name) - if dst.name != src.name: - sources.add(src.name) - self.copies.append(CopyEffect(src, dst)) + src = pred.pokes.pop(-1) + dst = self.peeks.pop(0) + assert src.offset.equivalent_to(dst.offset), (src, dst) + pred.final_offset.deeper(src.effect) + if dst.effect.name != src.effect.name: + if dst.effect.name != UNUSED: + destinations.add(dst.effect.name) + if src.effect.name != UNUSED: + sources.add(src.effect.name) + self.copies.append(CopyItem(src, dst)) # TODO: Turn this into an error (pass an Analyzer instance?) assert sources & destinations == set(), ( pred.instr.name, @@ -202,11 +225,27 @@ def __init__( else: pred = None # Break + # Fix up patterns of copies through UNUSED, + # e.g. cp(a, UNUSED) + cp(UNUSED, b) -> cp(a, b). + if any(copy.src.effect.name == UNUSED for copy in self.copies): + pred = self.pred + while pred is not None: + for copy in self.copies: + if copy.src.effect.name == UNUSED: + for pred_copy in pred.copies: + if pred_copy.dst == copy.src: + copy.src = pred_copy.src + break + pred = pred.pred + def adjust_deeper(self, eff: StackEffect) -> None: for peek in self.peeks: peek.offset.deeper(eff) for poke in self.pokes: poke.offset.deeper(eff) + for copy in self.copies: + copy.src.offset.deeper(eff) + copy.dst.offset.deeper(eff) self.min_offset.deeper(eff) self.final_offset.deeper(eff) @@ -215,6 +254,9 @@ def adjust_higher(self, eff: StackEffect) -> None: peek.offset.higher(eff) for poke in self.pokes: poke.offset.higher(eff) + for copy in self.copies: + copy.src.offset.higher(eff) + copy.dst.offset.higher(eff) self.min_offset.higher(eff) self.final_offset.higher(eff) @@ -248,8 +290,8 @@ def add(eff: StackEffect) -> None: vars[eff.name] = eff for copy in self.copies: - add(copy.src) - add(copy.dst) + add(copy.src.effect) + add(copy.dst.effect) for peek in self.peeks: add(peek.effect) for poke in self.pokes: @@ -365,8 +407,11 @@ def write_components( out.emit(f"// {mgr.instr.name}") for copy in mgr.copies: - if copy.src.name != copy.dst.name: - out.assign(copy.dst, copy.src) + copy_src_effect = copy.src.effect + if copy_src_effect.name != copy.dst.effect.name: + if copy_src_effect.name == UNUSED: + copy_src_effect = copy.src.as_stack_effect() + out.assign(copy.dst.effect, copy_src_effect) for peek in mgr.peeks: out.assign( peek.effect, From c3d580b238fb1b5a72d5608ff7905e9ad726d1bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 21:44:34 +0200 Subject: [PATCH 311/598] gh-106320: Remove private _PyList functions (#108451) Move private functions to the internal C API (pycore_list.h): * _PyList_Extend() * _PyList_DebugMallocStats() No longer export these functions. --- Include/cpython/listobject.h | 3 --- Include/internal/pycore_list.h | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index b3b23985de7a66..661610548733fd 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -21,9 +21,6 @@ typedef struct { Py_ssize_t allocated; } PyListObject; -PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); -PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); - /* Cast argument to PyListObject* type. */ #define _PyList_CAST(op) \ (assert(PyList_Check(op)), _Py_CAST(PyListObject*, (op))) diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index b2e503c87542bf..056be2c80c8ce6 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,7 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "listobject.h" // _PyList_CAST() + +extern PyObject* _PyList_Extend(PyListObject *, PyObject *); +extern void _PyList_DebugMallocStats(FILE *out); /* runtime lifecycle */ From 26893016a7f204b2e7138fc9ce04525a651c202e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 22:01:50 +0200 Subject: [PATCH 312/598] gh-106320: Remove private _PyDict functions (#108449) Move private functions to the internal C API (pycore_dict.h): * _PyDictView_Intersect() * _PyDictView_New() * _PyDict_ContainsId() * _PyDict_DelItemId() * _PyDict_DelItem_KnownHash() * _PyDict_GetItemIdWithError() * _PyDict_GetItem_KnownHash() * _PyDict_HasSplitTable() * _PyDict_NewPresized() * _PyDict_Next() * _PyDict_Pop() * _PyDict_SetItemId() * _PyDict_SetItem_KnownHash() * _PyDict_SizeOf() No longer export most of these functions. Move also the _PyDictViewObject structure to the internal C API. Move dict_getitem_knownhash() function from _testcapi to the _testinternalcapi extension. Update test_capi.test_dict for this change. --- Include/cpython/dictobject.h | 30 ---------------------- Include/internal/pycore_dict.h | 46 ++++++++++++++++++++++++++++++++-- Lib/test/test_dict.py | 4 +-- Modules/_asynciomodule.c | 1 + Modules/_collectionsmodule.c | 1 + Modules/_ctypes/stgdict.c | 1 + Modules/_sre/sre.c | 1 + Modules/_testcapimodule.c | 21 ---------------- Modules/_testinternalcapi.c | 22 ++++++++++++++++ Modules/_threadmodule.c | 4 +-- Modules/socketmodule.c | 1 + Python/import.c | 1 + 12 files changed, 76 insertions(+), 57 deletions(-) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index f44880991f4136..b05ca3ef453816 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,19 +32,8 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; -PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, - PyObject *item, Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); - -PyAPI_FUNC(int) _PyDict_Next( - PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { @@ -56,26 +45,7 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); -PyAPI_FUNC(int) _PyDict_ContainsId(PyObject *, _Py_Identifier *); - -PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *, PyObject *, PyObject *); -#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) - -PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); - -PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); - -/* _PyDictView */ - -typedef struct { - PyObject_HEAD - PyDictObject *dv_dict; -} _PyDictViewObject; -PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); /* Dictionary watchers */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index df7bc7e58f6c97..e79fb207cf75b1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -14,15 +14,45 @@ extern "C" { // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); - extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value)); +// "KnownHash" variants +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, + PyObject *item, Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); +extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); + +// "Id" variants +extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, + _Py_Identifier *key); +extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); +extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); +extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); + +extern int _PyDict_Next( + PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); + extern int _PyDict_HasOnlyStringKeys(PyObject *mp); extern void _PyDict_MaybeUntrack(PyObject *mp); +extern PyObject* _PyDict_NewPresized(Py_ssize_t minused); + +// Export for '_ctypes' shared extension +PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); + +// Export for '_socket' shared extension (Windows remove_unusable_flags()) +PyAPI_FUNC(PyObject*) _PyDict_Pop(PyObject *, PyObject *, PyObject *); + +#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) + /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, the first occurrence of a key wins, if override is 1, the last occurrence of a key wins, if override is 2, a KeyError with conflicting key as @@ -32,6 +62,18 @@ extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); extern void _PyDict_DebugMallocStats(FILE *out); + +/* _PyDictView */ + +typedef struct { + PyObject_HEAD + PyDictObject *dv_dict; +} _PyDictViewObject; + +extern PyObject* _PyDictView_New(PyObject *, PyTypeObject *); +extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); + + /* runtime lifecycle */ extern void _PyDict_Fini(PyInterpreterState *interp); diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index fbc6ce8282de3c..eab64b4f9106c1 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1582,8 +1582,8 @@ class CAPITest(unittest.TestCase): # Test _PyDict_GetItem_KnownHash() @support.cpython_only def test_getitem_knownhash(self): - _testcapi = import_helper.import_module('_testcapi') - dict_getitem_knownhash = _testcapi.dict_getitem_knownhash + _testinternalcapi = import_helper.import_module('_testinternalcapi') + dict_getitem_knownhash = _testinternalcapi.dict_getitem_knownhash d = {'x': 1, 'y': 2, 'z': 3} self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 6266dc8e3555f8..c66a8623413f4b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3,6 +3,7 @@ #endif #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index f2915f83b9d968..c8cd53de5e2262 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_typeobject.h" // _PyType_GetModuleState() diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 3348ebd6593d2f..54291a71667e38 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -9,6 +9,7 @@ #endif #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_SizeOf() #include #ifdef MS_WIN32 # include diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index c4e43a0db0f5d3..3872c3663c7294 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -39,6 +39,7 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #include "Python.h" +#include "pycore_dict.h" // _PyDict_Next() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6a1eba3a0de217..20b96320f4c339 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -271,26 +271,6 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -static PyObject* -dict_getitem_knownhash(PyObject *self, PyObject *args) -{ - PyObject *mp, *key, *result; - Py_ssize_t hash; - - if (!PyArg_ParseTuple(args, "OOn:dict_getitem_knownhash", - &mp, &key, &hash)) { - return NULL; - } - - result = _PyDict_GetItem_KnownHash(mp, key, (Py_hash_t)hash); - if (result == NULL && !PyErr_Occurred()) { - _PyErr_SetKeyError(key); - return NULL; - } - - return Py_XNewRef(result); -} - /* Issue #4701: Check that PyObject_Hash implicitly calls * PyType_Ready if it hasn't already been called */ @@ -3353,7 +3333,6 @@ static PyMethodDef TestMethods[] = { {"test_sizeof_c_types", test_sizeof_c_types, METH_NOARGS}, {"test_list_api", test_list_api, METH_NOARGS}, {"test_dict_iteration", test_dict_iteration, METH_NOARGS}, - {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {"test_lazy_hash_inheritance", test_lazy_hash_inheritance,METH_NOARGS}, {"test_xincref_doesnt_leak",test_xincref_doesnt_leak, METH_NOARGS}, {"test_incref_doesnt_leak", test_incref_doesnt_leak, METH_NOARGS}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 9b45d59987dca6..3e3dfeca037c7f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1572,6 +1572,27 @@ new_hamt(PyObject *self, PyObject *args) } +static PyObject* +dict_getitem_knownhash(PyObject *self, PyObject *args) +{ + PyObject *mp, *key, *result; + Py_ssize_t hash; + + if (!PyArg_ParseTuple(args, "OOn:dict_getitem_knownhash", + &mp, &key, &hash)) { + return NULL; + } + + result = _PyDict_GetItem_KnownHash(mp, key, (Py_hash_t)hash); + if (result == NULL && !PyErr_Occurred()) { + _PyErr_SetKeyError(key); + return NULL; + } + + return Py_XNewRef(result); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1637,6 +1658,7 @@ static PyMethodDef module_functions[] = { {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, + {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 984747c44d7a57..229abfba82d1ad 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -4,14 +4,14 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_MakePendingCalls() +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#include // offsetof() - +#include // offsetof() #ifdef HAVE_SIGNAL_H # include // SIGINT #endif diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 392a56d5ab7d3e..d1ac1ffec844a0 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -106,6 +106,7 @@ Local naming conventions: #endif #include "Python.h" +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState diff --git a/Python/import.c b/Python/import.c index 5681deb3c4fc73..48090d05240188 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,6 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() From a071ecb4d13595f3580cf82061dcd7b39cd475c5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 22:02:09 +0200 Subject: [PATCH 313/598] gh-106320: Remove private _PySys functions (#108452) Move private functions to the internal C API (pycore_sysmodule.h): * _PySys_GetAttr() * _PySys_GetSizeOf() No longer export most of these functions. Fix also a typo in Include/cpython/optimizer.h: add a missing space. --- Include/cpython/optimizer.h | 2 +- Include/cpython/sysmodule.h | 5 ----- Include/internal/pycore_sysmodule.h | 6 ++++++ Modules/_io/bytesio.c | 2 ++ Modules/_pickle.c | 3 ++- Modules/_threadmodule.c | 1 + Modules/faulthandler.c | 1 + Python/_warnings.c | 2 ++ Python/bltinmodule.c | 1 + Python/intrinsics.c | 3 ++- Python/sysmodule.c | 1 + Python/traceback.c | 1 + 12 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index e3fe0e8f0ffd59..10457afc180a00 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -38,7 +38,7 @@ PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); -PyAPI_FUNC(_PyExecutorObject *)PyUnstable_GetExecutor(PyCodeObject *code, int offset); +PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); struct _PyInterpreterFrame * _PyOptimizer_BackEdge(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer); diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h index 19d9dddc344a4f..e028fb7ecd5230 100644 --- a/Include/cpython/sysmodule.h +++ b/Include/cpython/sysmodule.h @@ -2,11 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(PyObject *) _PySys_GetAttr(PyThreadState *tstate, - PyObject *name); - -PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); - typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); PyAPI_FUNC(int) PySys_Audit( diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index aec9c2060fd0c0..9b8eafd3d6cfd3 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +// Export for '_pickle' shared extension +PyAPI_FUNC(PyObject*) _PySys_GetAttr(PyThreadState *tstate, PyObject *name); + +// Export for '_pickle' shared extension +PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); + extern int _PySys_Audit( PyThreadState *tstate, const char *event, diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 3ab503c9e3998d..f3074203f54ea2 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,5 +1,7 @@ #include "Python.h" #include "pycore_object.h" +#include "pycore_sysmodule.h" // _PySys_GetSizeOf() + #include // offsetof() #include "_iomodule.h" diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 4f26ffe27d75f1..b97524856eeca8 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -17,10 +17,11 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_setobject.h" // _PySet_NextEntry() - +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include // strtol() + PyDoc_STRVAR(pickle_module_doc, "Optimized C implementation for the Python pickle module."); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 229abfba82d1ad..2cf866c5a11416 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -9,6 +9,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include // offsetof() diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index d8cfc13a7dc6d5..5ec34d4e99c80d 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -3,6 +3,7 @@ #include "pycore_pyerrors.h" // _Py_DumpExtensionModules #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // Py_NSIG +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #include diff --git a/Python/_warnings.c b/Python/_warnings.c index 40ec5f613d5bf4..9e562d7ba7f6a7 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -7,6 +7,8 @@ #include "pycore_pyerrors.h" #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_sysmodule.h" // _PySys_GetAttr() + #include "clinic/_warnings.c.h" #define MODULE_NAME "_warnings" diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 787f53fae354db..ac9bc72fe80d6c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -12,6 +12,7 @@ #include "pycore_object.h" // _Py_AddToAllObjects() #include "pycore_pyerrors.h" // _PyErr_NoMemory() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "clinic/bltinmodule.c.h" diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 5267c10238e626..fefee0fc4ebbec 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -4,10 +4,11 @@ #include "Python.h" #include "pycore_frame.h" #include "pycore_function.h" -#include "pycore_runtime.h" #include "pycore_global_objects.h" #include "pycore_intrinsics.h" #include "pycore_pyerrors.h" +#include "pycore_runtime.h" +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_typevarobject.h" diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 94d0f01f7477a7..0ec763c7aa7cf8 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -31,6 +31,7 @@ Data members: #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_structseq.h" // _PyStructSequence_InitBuiltinWithFlags() +#include "pycore_sysmodule.h" // Define _PySys_GetSizeOf() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "frameobject.h" // PyFrame_FastToLocalsWithError() diff --git a/Python/traceback.c b/Python/traceback.c index 61ace38275355c..657ddab1cbf615 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -13,6 +13,7 @@ #include "pycore_pyarena.h" // _PyArena_Free() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // EXCEPTION_TB_HEADER #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() From edb569e8e1f469ac5ce629cc647ab169d895de41 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Thu, 24 Aug 2023 22:37:20 +0200 Subject: [PATCH 314/598] gh-108447: Detect platform triplets for x86_64 GNU/Hurd (#108045) --- .../Build/2023-08-24-18-36-31.gh-issue-108447.Ofsygr.rst | 1 + Misc/platform_triplet.c | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2023-08-24-18-36-31.gh-issue-108447.Ofsygr.rst diff --git a/Misc/NEWS.d/next/Build/2023-08-24-18-36-31.gh-issue-108447.Ofsygr.rst b/Misc/NEWS.d/next/Build/2023-08-24-18-36-31.gh-issue-108447.Ofsygr.rst new file mode 100644 index 00000000000000..a695ef503e406e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-24-18-36-31.gh-issue-108447.Ofsygr.rst @@ -0,0 +1 @@ +Fix x86_64 GNU/Hurd build diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index c5e64b93553a7c..3307260544e8a6 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -225,7 +225,13 @@ PLATFORM_TRIPLET=i386-kfreebsd-gnu # error unknown platform triplet # endif #elif defined(__gnu_hurd__) +# if defined(__x86_64__) && defined(__LP64__) +PLATFORM_TRIPLET=x86_64-gnu +# elif defined(__i386__) PLATFORM_TRIPLET=i386-gnu +# else +# error unknown platform triplet +# endif #elif defined(__APPLE__) PLATFORM_TRIPLET=darwin #elif defined(__VXWORKS__) From feb9a49c9c09d08cb8c24cb74d90a218de6af244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 24 Aug 2023 22:05:03 +0100 Subject: [PATCH 315/598] GH-103247: bypass the import cache on the _require_loader helper --- Lib/test/test_import/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 051711bfd1fe24..559198759c0d23 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -25,6 +25,7 @@ import _testinternalcapi import _imp +from test.support import import_helper from test.support import os_helper from test.support import ( STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, @@ -58,7 +59,7 @@ def _require_loader(module, loader, skip): if isinstance(module, str): - module = __import__(module) + module = import_helper.import_fresh_module(module) MODULE_KINDS = { BuiltinImporter: 'built-in', From be436e08b8bd9fcd2202d6ce4d924bba7551e96f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 23:55:30 +0200 Subject: [PATCH 316/598] gh-108444: Add PyLong_AsInt() public function (#108445) * Rename _PyLong_AsInt() to PyLong_AsInt(). * Add documentation. * Add test. * For now, keep _PyLong_AsInt() as an alias to PyLong_AsInt(). --- Doc/c-api/long.rst | 8 +++++ Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.13.rst | 6 ++++ Include/cpython/longobject.h | 3 +- Include/longobject.h | 6 ++++ Lib/test/test_capi/test_long.py | 30 +++++++++++++++++++ Lib/test/test_stable_abi_ctypes.py | 1 + ...-08-24-20-08-02.gh-issue-108014.20DOSS.rst | 4 +++ Misc/stable_abi.toml | 2 ++ Modules/_testcapi/clinic/long.c.h | 10 ++++++- Modules/_testcapi/long.c | 25 ++++++++++++++++ Objects/longobject.c | 2 +- PC/python3dll.c | 1 + 13 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-08-24-20-08-02.gh-issue-108014.20DOSS.rst diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f1354a34f2b2d5..045604870d3c84 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -136,6 +136,14 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. This function will no longer use :meth:`~object.__int__`. +.. c:function:: int PyLong_AsInt(PyObject *obj) + + Similar to :c:func:`PyLong_AsLong`, but store the result in a C + :c:expr:`int` instead of a C :c:expr:`long`. + + .. versionadded:: 3.13 + + .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index ed415a4dc644a4..cc6349a0330e8c 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -347,6 +347,7 @@ var,PyList_Type,3.2,, type,PyLongObject,3.2,,opaque var,PyLongRangeIter_Type,3.2,, function,PyLong_AsDouble,3.2,, +function,PyLong_AsInt,3.13,, function,PyLong_AsLong,3.2,, function,PyLong_AsLongAndOverflow,3.2,, function,PyLong_AsLongLong,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 25eb5e981c55e1..4ff12b16d00266 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -871,6 +871,12 @@ New Features :term:`shutting down `. (Contributed by Victor Stinner in :gh:`108014`.) +* Add :c:func:`PyLong_AsInt` function: similar to :c:func:`PyLong_AsLong`, but + store the result in a C :c:expr:`int` instead of a C :c:expr:`long`. + Previously, it was known as the private function :c:func:`!_PyLong_AsInt` + (with an underscore prefix). + (Contributed by Victor Stinner in :gh:`108014`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index c581f51cbcdf38..c96f35188c47fe 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,7 +2,8 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); +// Alias for backport compatibility +#define _PyLong_AsInt PyLong_AsInt PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); diff --git a/Include/longobject.h b/Include/longobject.h index e559e238ae5a35..7393254cd24a9b 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -18,12 +18,18 @@ PyAPI_FUNC(PyObject *) PyLong_FromUnsignedLong(unsigned long); PyAPI_FUNC(PyObject *) PyLong_FromSize_t(size_t); PyAPI_FUNC(PyObject *) PyLong_FromSsize_t(Py_ssize_t); PyAPI_FUNC(PyObject *) PyLong_FromDouble(double); + PyAPI_FUNC(long) PyLong_AsLong(PyObject *); PyAPI_FUNC(long) PyLong_AsLongAndOverflow(PyObject *, int *); PyAPI_FUNC(Py_ssize_t) PyLong_AsSsize_t(PyObject *); PyAPI_FUNC(size_t) PyLong_AsSize_t(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLong(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PyLong_AsInt(PyObject *); +#endif + PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 8928fd94a1d6a3..101fe1f0de77f1 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -34,6 +34,36 @@ def test_compact_known(self): self.assertEqual(_testcapi.call_long_compact_api(sys.maxsize), (False, -1)) + def test_long_asint(self): + PyLong_AsInt = _testcapi.PyLong_AsInt + INT_MIN = _testcapi.INT_MIN + INT_MAX = _testcapi.INT_MAX + + # round trip (object -> int -> object) + for value in (INT_MIN, INT_MAX, -1, 0, 1, 123): + with self.subTest(value=value): + self.assertEqual(PyLong_AsInt(value), value) + + # use __index__(), not __int__() + class MyIndex: + def __index__(self): + return 10 + def __int__(self): + return 22 + self.assertEqual(PyLong_AsInt(MyIndex()), 10) + + # bound checking + with self.assertRaises(OverflowError): + PyLong_AsInt(INT_MIN - 1) + with self.assertRaises(OverflowError): + PyLong_AsInt(INT_MAX + 1) + + # invalid type + for value in (1.0, b'2', '3'): + with self.subTest(value=value): + with self.assertRaises(TypeError): + PyLong_AsInt(value) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 566d36a3f5ba11..1f3cf612c18f6d 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -375,6 +375,7 @@ def test_windows_feature_macros(self): "PyList_Type", "PyLongRangeIter_Type", "PyLong_AsDouble", + "PyLong_AsInt", "PyLong_AsLong", "PyLong_AsLongAndOverflow", "PyLong_AsLongLong", diff --git a/Misc/NEWS.d/next/C API/2023-08-24-20-08-02.gh-issue-108014.20DOSS.rst b/Misc/NEWS.d/next/C API/2023-08-24-20-08-02.gh-issue-108014.20DOSS.rst new file mode 100644 index 00000000000000..5c1b04f3237e78 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-24-20-08-02.gh-issue-108014.20DOSS.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyLong_AsInt` function: similar to :c:func:`PyLong_AsLong`, but +store the result in a C :c:expr:`int` instead of a C :c:expr:`long`. +Previously, it was known as the the private function :c:func:`!_PyLong_AsInt` +(with an underscore prefix). Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 16d5c1a07ae3e2..2030a085abf27c 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2450,3 +2450,5 @@ added = '3.13' [function.PyDict_GetItemStringRef] added = '3.13' +[function.PyLong_AsInt] + added = '3.13' diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index 95885e0ae17a6c..87bba4cfacfe55 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -163,4 +163,12 @@ PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, #define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, -/*[clinic end generated code: output=d000a1b58fa81eab input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, +"PyLong_AsInt($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_PYLONG_ASINT_METHODDEF \ + {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, +/*[clinic end generated code: output=1631a18f1193486a input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index ede43f60d06f95..6b74e0ab8e0d1c 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -37,6 +37,9 @@ raise_test_long_error(const char* msg) return raiseTestError("test_long_api", msg); } +// Test PyLong_FromLong()/PyLong_AsLong() +// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). + #define TESTNAME test_long_api_inner #define TYPENAME long #define F_S_TO_PY PyLong_FromLong @@ -64,6 +67,9 @@ _testcapi_test_long_api_impl(PyObject *module) #undef F_U_TO_PY #undef F_PY_TO_U +// Test PyLong_FromLongLong()/PyLong_AsLongLong() +// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). + static PyObject * raise_test_longlong_error(const char* msg) { @@ -595,6 +601,24 @@ _testcapi_call_long_compact_api(PyObject *module, PyObject *arg) return Py_BuildValue("in", is_compact, value); } +/*[clinic input] +_testcapi.PyLong_AsInt + arg: object + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_PyLong_AsInt(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=0df9f19de5fa575b input=9561b97105493a67]*/ +{ + assert(!PyErr_Occurred()); + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + static PyMethodDef test_methods[] = { _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF _TESTCAPI_TEST_LONG_API_METHODDEF @@ -605,6 +629,7 @@ static PyMethodDef test_methods[] = { _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF _TESTCAPI_TEST_LONGLONG_API_METHODDEF _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF + _TESTCAPI_PYLONG_ASINT_METHODDEF {NULL}, }; diff --git a/Objects/longobject.c b/Objects/longobject.c index 354cba9d6d800f..d20ef412367bb7 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -549,7 +549,7 @@ PyLong_AsLong(PyObject *obj) method. Return -1 and set an error if overflow occurs. */ int -_PyLong_AsInt(PyObject *obj) +PyLong_AsInt(PyObject *obj) { int overflow; long result = PyLong_AsLongAndOverflow(obj, &overflow); diff --git a/PC/python3dll.c b/PC/python3dll.c index 64dfbba3e424a1..ee3a7d7b4e5be1 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -331,6 +331,7 @@ EXPORT_FUNC(PyList_SetSlice) EXPORT_FUNC(PyList_Size) EXPORT_FUNC(PyList_Sort) EXPORT_FUNC(PyLong_AsDouble) +EXPORT_FUNC(PyLong_AsInt) EXPORT_FUNC(PyLong_AsLong) EXPORT_FUNC(PyLong_AsLongAndOverflow) EXPORT_FUNC(PyLong_AsLongLong) From be800f4be78106d7566c694b3a5652761798db96 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 25 Aug 2023 01:05:23 +0300 Subject: [PATCH 317/598] Run `mypy` when `Tools/requirements-dev.txt` changes (#108457) --- .github/workflows/mypy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index a83a90c69529f2..75264c8e746663 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -9,6 +9,7 @@ on: paths: - "Tools/clinic/**" - "Tools/cases_generator/**" + - "Tools/requirements-dev.txt" - ".github/workflows/mypy.yml" workflow_dispatch: From 4e5a7284eef4308dce252ca1115d4a5a5b7e6ae8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Aug 2023 00:51:22 +0200 Subject: [PATCH 318/598] gh-108444: Argument Clinic uses PyLong_AsInt() (#108458) Argument Clinic now uses the new public PyLong_AsInt(), rather than the old name _PyLong_AsInt(). --- Lib/test/clinic.test.c | 24 +-- Lib/test/test_clinic.py | 5 +- Modules/_blake2/clinic/blake2b_impl.c.h | 12 +- Modules/_blake2/clinic/blake2s_impl.c.h | 12 +- Modules/_io/clinic/_iomodule.c.h | 6 +- Modules/_io/clinic/bufferedio.c.h | 8 +- Modules/_io/clinic/bytesio.c.h | 4 +- Modules/_io/clinic/fileio.c.h | 4 +- Modules/_io/clinic/iobase.c.h | 6 +- Modules/_io/clinic/stringio.c.h | 4 +- Modules/_io/clinic/textio.c.h | 8 +- .../clinic/multiprocessing.c.h | 4 +- .../_multiprocessing/clinic/posixshmem.c.h | 6 +- Modules/_multiprocessing/clinic/semaphore.c.h | 8 +- Modules/_sqlite/clinic/blob.c.h | 8 +- Modules/_sqlite/clinic/connection.c.h | 26 +-- Modules/_sqlite/clinic/cursor.c.h | 4 +- Modules/_sqlite/clinic/module.c.h | 4 +- Modules/_sre/clinic/sre.c.h | 12 +- Modules/_ssl/clinic/cert.c.h | 4 +- Modules/_testcapi/clinic/exceptions.c.h | 4 +- Modules/_testcapi/clinic/float.c.h | 6 +- Modules/_testcapi/clinic/watchers.c.h | 10 +- Modules/clinic/_bz2module.c.h | 4 +- Modules/clinic/_codecsmodule.c.h | 14 +- Modules/clinic/_curses_panel.c.h | 6 +- Modules/clinic/_cursesmodule.c.h | 66 +++--- Modules/clinic/_datetimemodule.c.h | 8 +- Modules/clinic/_dbmmodule.c.h | 4 +- Modules/clinic/_gdbmmodule.c.h | 4 +- Modules/clinic/_localemodule.c.h | 8 +- Modules/clinic/_lzmamodule.c.h | 6 +- Modules/clinic/_opcode.c.h | 20 +- Modules/clinic/_posixsubprocess.c.h | 20 +- Modules/clinic/_randommodule.c.h | 4 +- Modules/clinic/_ssl.c.h | 10 +- Modules/clinic/_testclinic.c.h | 8 +- Modules/clinic/_testinternalcapi.c.h | 8 +- Modules/clinic/_testmultiphase.c.h | 4 +- Modules/clinic/_tkinter.c.h | 12 +- Modules/clinic/_tracemalloc.c.h | 4 +- Modules/clinic/arraymodule.c.h | 4 +- Modules/clinic/binascii.c.h | 6 +- Modules/clinic/fcntlmodule.c.h | 10 +- Modules/clinic/gcmodule.c.h | 6 +- Modules/clinic/overlapped.c.h | 10 +- Modules/clinic/posixmodule.c.h | 194 +++++++++--------- Modules/clinic/pyexpat.c.h | 4 +- Modules/clinic/readline.c.h | 12 +- Modules/clinic/resource.c.h | 10 +- Modules/clinic/selectmodule.c.h | 14 +- Modules/clinic/signalmodule.c.h | 32 +-- Modules/clinic/socketmodule.c.h | 8 +- Modules/clinic/termios.c.h | 10 +- Modules/clinic/zlibmodule.c.h | 22 +- Objects/clinic/bytearrayobject.c.h | 6 +- Objects/clinic/bytesobject.c.h | 4 +- Objects/clinic/codeobject.c.h | 32 +-- Objects/clinic/memoryobject.c.h | 6 +- Objects/clinic/typeobject.c.h | 4 +- Objects/clinic/unicodeobject.c.h | 4 +- Objects/stringlib/clinic/transmogrify.h.h | 4 +- PC/clinic/msvcrtmodule.c.h | 22 +- PC/clinic/winreg.c.h | 22 +- PC/clinic/winsound.c.h | 10 +- Python/clinic/_warnings.c.h | 4 +- Python/clinic/bltinmodule.c.h | 12 +- Python/clinic/import.c.h | 6 +- Python/clinic/instrumentation.c.h | 24 +-- Python/clinic/marshal.c.h | 6 +- Python/clinic/sysmodule.c.h | 16 +- Python/clinic/traceback.c.h | 6 +- Tools/clinic/clinic.py | 4 +- 73 files changed, 467 insertions(+), 466 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 019dc10073e986..2ef7a6e8eb2795 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -473,7 +473,7 @@ test_bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - c = _PyLong_AsInt(args[2]); + c = PyLong_AsInt(args[2]); if (c == -1 && PyErr_Occurred()) { goto exit; } @@ -486,7 +486,7 @@ test_bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_bool_converter_impl(PyObject *module, int a, int b, int c) -/*[clinic end generated code: output=27f0e653a70b9be3 input=939854fa9f248c60]*/ +/*[clinic end generated code: output=3190e46490de0644 input=939854fa9f248c60]*/ /*[clinic input] @@ -1009,14 +1009,14 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - a = _PyLong_AsInt(args[0]); + a = PyLong_AsInt(args[0]); if (a == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 2) { goto skip_optional; } - b = _PyLong_AsInt(args[1]); + b = PyLong_AsInt(args[1]); if (b == -1 && PyErr_Occurred()) { goto exit; } @@ -1035,7 +1035,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - d = _PyLong_AsInt(args[3]); + d = PyLong_AsInt(args[3]); if (d == -1 && PyErr_Occurred()) { goto exit; } @@ -1048,7 +1048,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_int_converter_impl(PyObject *module, int a, int b, int c, myenum d) -/*[clinic end generated code: output=375eedba5ca9a5b3 input=d20541fc1ca0553e]*/ +/*[clinic end generated code: output=5aed87a7589eefb2 input=d20541fc1ca0553e]*/ /*[clinic input] @@ -4509,7 +4509,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ if (!args) { goto exit; } - a = _PyLong_AsInt(args[0]); + a = PyLong_AsInt(args[0]); if (a == -1 && PyErr_Occurred()) { goto exit; } @@ -4521,7 +4521,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ static PyObject * Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) -/*[clinic end generated code: output=00218e7f583e6c81 input=af158077bd237ef9]*/ +/*[clinic end generated code: output=d89b99e83d442be0 input=af158077bd237ef9]*/ /*[clinic input] @@ -4703,7 +4703,7 @@ Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) PyObject *return_value = NULL; int arg; - arg = _PyLong_AsInt(arg_); + arg = PyLong_AsInt(arg_); if (arg == -1 && PyErr_Occurred()) { goto exit; } @@ -4715,7 +4715,7 @@ Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) static PyObject * Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) -/*[clinic end generated code: output=7d590626642194ae input=2a53a57cf5624f95]*/ +/*[clinic end generated code: output=9f04de4a62287e28 input=2a53a57cf5624f95]*/ /*[clinic input] @@ -5062,7 +5062,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t if (!args) { goto exit; } - int_value = _PyLong_AsInt(args[0]); + int_value = PyLong_AsInt(args[0]); if (int_value == -1 && PyErr_Occurred()) { goto exit; } @@ -5074,7 +5074,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * mangled_c_keyword_identifier_impl(PyObject *module, int int_value) -/*[clinic end generated code: output=c049d7d79be26cda input=060876448ab567a2]*/ +/*[clinic end generated code: output=f24b37e0368e0eb8 input=060876448ab567a2]*/ /*[clinic input] diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 310615083778f7..f61a10b89a777a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2463,11 +2463,12 @@ def test_cli_force(self): # Verify by checking the checksum. checksum = ( "/*[clinic end generated code: " - "output=2124c291eb067d76 input=9543a8d2da235301]*/\n" + "output=c16447c01510dfb3 input=9543a8d2da235301]*/\n" ) with open(fn, 'r', encoding='utf-8') as f: generated = f.read() - self.assertTrue(generated.endswith(checksum)) + self.assertTrue(generated.endswith(checksum), + (generated, checksum)) def test_cli_make(self): c_code = dedent(""" diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 99b0f098cc2b27..284cccf348703c 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -85,7 +85,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } if (fastargs[1]) { - digest_size = _PyLong_AsInt(fastargs[1]); + digest_size = PyLong_AsInt(fastargs[1]); if (digest_size == -1 && PyErr_Occurred()) { goto exit; } @@ -130,7 +130,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[5]) { - fanout = _PyLong_AsInt(fastargs[5]); + fanout = PyLong_AsInt(fastargs[5]); if (fanout == -1 && PyErr_Occurred()) { goto exit; } @@ -139,7 +139,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[6]) { - depth = _PyLong_AsInt(fastargs[6]); + depth = PyLong_AsInt(fastargs[6]); if (depth == -1 && PyErr_Occurred()) { goto exit; } @@ -164,7 +164,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[9]) { - node_depth = _PyLong_AsInt(fastargs[9]); + node_depth = PyLong_AsInt(fastargs[9]); if (node_depth == -1 && PyErr_Occurred()) { goto exit; } @@ -173,7 +173,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[10]) { - inner_size = _PyLong_AsInt(fastargs[10]); + inner_size = PyLong_AsInt(fastargs[10]); if (inner_size == -1 && PyErr_Occurred()) { goto exit; } @@ -276,4 +276,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=996b4fe396824797 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=76bbcf5f220511b9 input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index 9b821fbcd62cdb..5b6af84bb24234 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -85,7 +85,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } if (fastargs[1]) { - digest_size = _PyLong_AsInt(fastargs[1]); + digest_size = PyLong_AsInt(fastargs[1]); if (digest_size == -1 && PyErr_Occurred()) { goto exit; } @@ -130,7 +130,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[5]) { - fanout = _PyLong_AsInt(fastargs[5]); + fanout = PyLong_AsInt(fastargs[5]); if (fanout == -1 && PyErr_Occurred()) { goto exit; } @@ -139,7 +139,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[6]) { - depth = _PyLong_AsInt(fastargs[6]); + depth = PyLong_AsInt(fastargs[6]); if (depth == -1 && PyErr_Occurred()) { goto exit; } @@ -164,7 +164,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[9]) { - node_depth = _PyLong_AsInt(fastargs[9]); + node_depth = PyLong_AsInt(fastargs[9]); if (node_depth == -1 && PyErr_Occurred()) { goto exit; } @@ -173,7 +173,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[10]) { - inner_size = _PyLong_AsInt(fastargs[10]); + inner_size = PyLong_AsInt(fastargs[10]); if (inner_size == -1 && PyErr_Occurred()) { goto exit; } @@ -276,4 +276,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=bd0fb7639e450618 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af69b321be0b4a77 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 6bb2f7f212a44d..77c203681fca44 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -202,7 +202,7 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } } if (args[2]) { - buffering = _PyLong_AsInt(args[2]); + buffering = PyLong_AsInt(args[2]); if (buffering == -1 && PyErr_Occurred()) { goto exit; } @@ -332,7 +332,7 @@ _io_text_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - stacklevel = _PyLong_AsInt(args[1]); + stacklevel = PyLong_AsInt(args[1]); if (stacklevel == -1 && PyErr_Occurred()) { goto exit; } @@ -404,4 +404,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=6800c35366b1a5f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a4ceb802f3a7243f input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 4f40fdadf8ec85..b9b42cf3428573 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -162,7 +162,7 @@ _io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *arg if (nargs < 1) { goto skip_optional_posonly; } - size = _PyLong_AsInt(args[0]); + size = PyLong_AsInt(args[0]); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -216,7 +216,7 @@ _io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *ar if (nargs < 1) { goto skip_optional_posonly; } - size = _PyLong_AsInt(args[0]); + size = PyLong_AsInt(args[0]); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -721,7 +721,7 @@ _io__Buffered_seek(buffered *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - whence = _PyLong_AsInt(args[1]); + whence = PyLong_AsInt(args[1]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -1098,4 +1098,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b7ddf84a5bc2bf34 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e5b335441452653d input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 9550c8728c251e..52c9c2c5ecbb6e 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -427,7 +427,7 @@ _io_BytesIO_seek(bytesio *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - whence = _PyLong_AsInt(args[1]); + whence = PyLong_AsInt(args[1]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -538,4 +538,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=098584d485420b65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d4e7651002e14c6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 33a37a389d223d..9f5198d45ab819 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -430,7 +430,7 @@ _io_FileIO_seek(fileio *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - whence = _PyLong_AsInt(args[1]); + whence = PyLong_AsInt(args[1]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -536,4 +536,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=bef47b31b644996a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=65b9a5cc96d193b6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 773e0010477053..a539125a2a8fce 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -55,14 +55,14 @@ _io__IOBase_seek(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ss if (!args) { goto exit; } - offset = _PyLong_AsInt(args[0]); + offset = PyLong_AsInt(args[0]); if (offset == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 2) { goto skip_optional_posonly; } - whence = _PyLong_AsInt(args[1]); + whence = PyLong_AsInt(args[1]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -436,4 +436,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=301b22f8f75ce3dc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f064cfd54e3c1a5 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index d495dd10c16330..d0acfdb4bf3818 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -198,7 +198,7 @@ _io_StringIO_seek(stringio *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - whence = _PyLong_AsInt(args[1]); + whence = PyLong_AsInt(args[1]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -367,4 +367,4 @@ _io_StringIO_seekable(stringio *self, PyObject *Py_UNUSED(ignored)) { return _io_StringIO_seekable_impl(self); } -/*[clinic end generated code: output=533f20ae9b773126 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6f55dc1454aeb507 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 9bccd71e1d3ea6..d653cc87ae317a 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -75,7 +75,7 @@ _io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, P if (nargs < 1) { goto skip_optional_posonly; } - size = _PyLong_AsInt(args[0]); + size = PyLong_AsInt(args[0]); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -129,7 +129,7 @@ _io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *arg if (nargs < 1) { goto skip_optional_posonly; } - size = _PyLong_AsInt(args[0]); + size = PyLong_AsInt(args[0]); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -798,7 +798,7 @@ _io_TextIOWrapper_seek(textio *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - whence = _PyLong_AsInt(args[1]); + whence = PyLong_AsInt(args[1]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -975,4 +975,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=6bd981a58fcbc778 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=29b945b24287dd0c input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/multiprocessing.c.h b/Modules/_multiprocessing/clinic/multiprocessing.c.h index 885cd5c2fff8ea..e8eed0c0c4283f 100644 --- a/Modules/_multiprocessing/clinic/multiprocessing.c.h +++ b/Modules/_multiprocessing/clinic/multiprocessing.c.h @@ -66,7 +66,7 @@ _multiprocessing_recv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!handle && PyErr_Occurred()) { goto exit; } - size = _PyLong_AsInt(args[1]); + size = PyLong_AsInt(args[1]); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -172,4 +172,4 @@ _multiprocessing_sem_unlink(PyObject *module, PyObject *arg) #ifndef _MULTIPROCESSING_SEND_METHODDEF #define _MULTIPROCESSING_SEND_METHODDEF #endif /* !defined(_MULTIPROCESSING_SEND_METHODDEF) */ -/*[clinic end generated code: output=4a6afc67c1f5ec85 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eb078dff6b6595ff input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index dd40662ce9902d..265f46889d531b 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -68,14 +68,14 @@ _posixshmem_shm_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto exit; } path = args[0]; - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } if (!noptargs) { goto skip_optional_pos; } - mode = _PyLong_AsInt(args[2]); + mode = PyLong_AsInt(args[2]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -166,4 +166,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=9b67f98885757bc4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c022e2f7371fdece input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 35347169bc1591..c5bffc843570d8 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -250,15 +250,15 @@ _multiprocessing_SemLock(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!fastargs) { goto exit; } - kind = _PyLong_AsInt(fastargs[0]); + kind = PyLong_AsInt(fastargs[0]); if (kind == -1 && PyErr_Occurred()) { goto exit; } - value = _PyLong_AsInt(fastargs[1]); + value = PyLong_AsInt(fastargs[1]); if (value == -1 && PyErr_Occurred()) { goto exit; } - maxvalue = _PyLong_AsInt(fastargs[2]); + maxvalue = PyLong_AsInt(fastargs[2]); if (maxvalue == -1 && PyErr_Occurred()) { goto exit; } @@ -542,4 +542,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=dae57a702cc01512 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0fcadfdd8d6944be input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index f3d8a35be46138..7dda4d77c72ab1 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -57,7 +57,7 @@ blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - length = _PyLong_AsInt(args[0]); + length = PyLong_AsInt(args[0]); if (length == -1 && PyErr_Occurred()) { goto exit; } @@ -133,14 +133,14 @@ blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { goto exit; } - offset = _PyLong_AsInt(args[0]); + offset = PyLong_AsInt(args[0]); if (offset == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 2) { goto skip_optional; } - origin = _PyLong_AsInt(args[1]); + origin = PyLong_AsInt(args[1]); if (origin == -1 && PyErr_Occurred()) { goto exit; } @@ -219,4 +219,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=ad6a402f70e85977 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1783b816ccc3bb75 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index fe2196d0f7ee54..75a7df437bfb80 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -105,7 +105,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - detect_types = _PyLong_AsInt(fastargs[2]); + detect_types = PyLong_AsInt(fastargs[2]); if (detect_types == -1 && PyErr_Occurred()) { goto exit; } @@ -137,7 +137,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[6]) { - cache_size = _PyLong_AsInt(fastargs[6]); + cache_size = PyLong_AsInt(fastargs[6]); if (cache_size == -1 && PyErr_Occurred()) { goto exit; } @@ -482,7 +482,7 @@ pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - narg = _PyLong_AsInt(args[1]); + narg = PyLong_AsInt(args[1]); if (narg == -1 && PyErr_Occurred()) { goto exit; } @@ -565,7 +565,7 @@ create_window_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *c PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - num_params = _PyLong_AsInt(args[1]); + num_params = PyLong_AsInt(args[1]); if (num_params == -1 && PyErr_Occurred()) { goto exit; } @@ -644,7 +644,7 @@ pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cl PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - n_arg = _PyLong_AsInt(args[1]); + n_arg = PyLong_AsInt(args[1]); if (n_arg == -1 && PyErr_Occurred()) { goto exit; } @@ -764,7 +764,7 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject goto exit; } callable = args[0]; - n = _PyLong_AsInt(args[1]); + n = PyLong_AsInt(args[1]); if (n == -1 && PyErr_Occurred()) { goto exit; } @@ -1146,7 +1146,7 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *const *args, Py_ goto skip_optional_kwonly; } if (args[1]) { - pages = _PyLong_AsInt(args[1]); + pages = PyLong_AsInt(args[1]); if (pages == -1 && PyErr_Occurred()) { goto exit; } @@ -1538,11 +1538,11 @@ setlimit(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setlimit", nargs, 2, 2)) { goto exit; } - category = _PyLong_AsInt(args[0]); + category = PyLong_AsInt(args[0]); if (category == -1 && PyErr_Occurred()) { goto exit; } - limit = _PyLong_AsInt(args[1]); + limit = PyLong_AsInt(args[1]); if (limit == -1 && PyErr_Occurred()) { goto exit; } @@ -1573,7 +1573,7 @@ getlimit(pysqlite_Connection *self, PyObject *arg) PyObject *return_value = NULL; int category; - category = _PyLong_AsInt(arg); + category = PyLong_AsInt(arg); if (category == -1 && PyErr_Occurred()) { goto exit; } @@ -1608,7 +1608,7 @@ setconfig(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setconfig", nargs, 1, 2)) { goto exit; } - op = _PyLong_AsInt(args[0]); + op = PyLong_AsInt(args[0]); if (op == -1 && PyErr_Occurred()) { goto exit; } @@ -1648,7 +1648,7 @@ getconfig(pysqlite_Connection *self, PyObject *arg) int op; int _return_value; - op = _PyLong_AsInt(arg); + op = PyLong_AsInt(arg); if (op == -1 && PyErr_Occurred()) { goto exit; } @@ -1681,4 +1681,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=0ad9d55977a51b8f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bc31bec42067a8bf input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index 08b6a732b7bac5..1a21a1c41dafd8 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -223,7 +223,7 @@ pysqlite_cursor_fetchmany(pysqlite_Cursor *self, PyObject *const *args, Py_ssize if (!noptargs) { goto skip_optional_pos; } - maxrows = _PyLong_AsInt(args[0]); + maxrows = PyLong_AsInt(args[0]); if (maxrows == -1 && PyErr_Occurred()) { goto exit; } @@ -313,4 +313,4 @@ pysqlite_cursor_close(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl(self); } -/*[clinic end generated code: output=831f7bc5256526d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b56b3ddb3b6df8c6 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 94002b015fc779..82e628ababd4a1 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -159,7 +159,7 @@ pysqlite_enable_callback_trace(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int enable; - enable = _PyLong_AsInt(arg); + enable = PyLong_AsInt(arg); if (enable == -1 && PyErr_Occurred()) { goto exit; } @@ -208,4 +208,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e08e6856ae546e7b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c1d450089867b4bf input=a9049054013a1b77]*/ diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index da641081ce9e3c..a303a557ca5f46 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -53,7 +53,7 @@ _sre_ascii_iscased(PyObject *module, PyObject *arg) int character; int _return_value; - character = _PyLong_AsInt(arg); + character = PyLong_AsInt(arg); if (character == -1 && PyErr_Occurred()) { goto exit; } @@ -85,7 +85,7 @@ _sre_unicode_iscased(PyObject *module, PyObject *arg) int character; int _return_value; - character = _PyLong_AsInt(arg); + character = PyLong_AsInt(arg); if (character == -1 && PyErr_Occurred()) { goto exit; } @@ -117,7 +117,7 @@ _sre_ascii_tolower(PyObject *module, PyObject *arg) int character; int _return_value; - character = _PyLong_AsInt(arg); + character = PyLong_AsInt(arg); if (character == -1 && PyErr_Occurred()) { goto exit; } @@ -149,7 +149,7 @@ _sre_unicode_tolower(PyObject *module, PyObject *arg) int character; int _return_value; - character = _PyLong_AsInt(arg); + character = PyLong_AsInt(arg); if (character == -1 && PyErr_Occurred()) { goto exit; } @@ -1031,7 +1031,7 @@ _sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } pattern = args[0]; - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -1460,4 +1460,4 @@ _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=e3ba72156dd71572 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b84e62c87238f0e input=a9049054013a1b77]*/ diff --git a/Modules/_ssl/clinic/cert.c.h b/Modules/_ssl/clinic/cert.c.h index a052ab2086fd96..e2842263bec04d 100644 --- a/Modules/_ssl/clinic/cert.c.h +++ b/Modules/_ssl/clinic/cert.c.h @@ -59,7 +59,7 @@ _ssl_Certificate_public_bytes(PySSLCertificate *self, PyObject *const *args, Py_ if (!noptargs) { goto skip_optional_pos; } - format = _PyLong_AsInt(args[0]); + format = PyLong_AsInt(args[0]); if (format == -1 && PyErr_Occurred()) { goto exit; } @@ -86,4 +86,4 @@ _ssl_Certificate_get_info(PySSLCertificate *self, PyObject *Py_UNUSED(ignored)) { return _ssl_Certificate_get_info_impl(self); } -/*[clinic end generated code: output=82efada014f9b7fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=72e2bb139c64546c input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 16954a5ebf3e58..61a9103b97f5b4 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -299,7 +299,7 @@ _testcapi_raise_exception(PyObject *module, PyObject *const *args, Py_ssize_t na goto exit; } exc = args[0]; - num_args = _PyLong_AsInt(args[1]); + num_args = PyLong_AsInt(args[1]); if (num_args == -1 && PyErr_Occurred()) { goto exit; } @@ -488,4 +488,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=d574342d716e98b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6c4b7ad1cb1e0153 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/float.c.h b/Modules/_testcapi/clinic/float.c.h index c1dff2a9ac5789..84de17a8ee1762 100644 --- a/Modules/_testcapi/clinic/float.c.h +++ b/Modules/_testcapi/clinic/float.c.h @@ -31,7 +31,7 @@ _testcapi_float_pack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("float_pack", nargs, 3, 3)) { goto exit; } - size = _PyLong_AsInt(args[0]); + size = PyLong_AsInt(args[0]); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -45,7 +45,7 @@ _testcapi_float_pack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } } - le = _PyLong_AsInt(args[2]); + le = PyLong_AsInt(args[2]); if (le == -1 && PyErr_Occurred()) { goto exit; } @@ -85,4 +85,4 @@ _testcapi_float_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=083e5df26cd5fbeb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de6879d0f4987d79 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/watchers.c.h b/Modules/_testcapi/clinic/watchers.c.h index 975244bd59a36b..ce9c770a37cc5f 100644 --- a/Modules/_testcapi/clinic/watchers.c.h +++ b/Modules/_testcapi/clinic/watchers.c.h @@ -29,7 +29,7 @@ _testcapi_watch_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("watch_dict", nargs, 2, 2)) { goto exit; } - watcher_id = _PyLong_AsInt(args[0]); + watcher_id = PyLong_AsInt(args[0]); if (watcher_id == -1 && PyErr_Occurred()) { goto exit; } @@ -61,7 +61,7 @@ _testcapi_unwatch_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (!_PyArg_CheckPositional("unwatch_dict", nargs, 2, 2)) { goto exit; } - watcher_id = _PyLong_AsInt(args[0]); + watcher_id = PyLong_AsInt(args[0]); if (watcher_id == -1 && PyErr_Occurred()) { goto exit; } @@ -93,7 +93,7 @@ _testcapi_watch_type(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("watch_type", nargs, 2, 2)) { goto exit; } - watcher_id = _PyLong_AsInt(args[0]); + watcher_id = PyLong_AsInt(args[0]); if (watcher_id == -1 && PyErr_Occurred()) { goto exit; } @@ -125,7 +125,7 @@ _testcapi_unwatch_type(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (!_PyArg_CheckPositional("unwatch_type", nargs, 2, 2)) { goto exit; } - watcher_id = _PyLong_AsInt(args[0]); + watcher_id = PyLong_AsInt(args[0]); if (watcher_id == -1 && PyErr_Occurred()) { goto exit; } @@ -195,4 +195,4 @@ _testcapi_set_func_kwdefaults_via_capi(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=12c375089125d165 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b7564a84c5815b46 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index d7797d639ae32e..6ad2db3ce1dee6 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -102,7 +102,7 @@ _bz2_BZ2Compressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyTuple_GET_SIZE(args) < 1) { goto skip_optional; } - compresslevel = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); + compresslevel = PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); if (compresslevel == -1 && PyErr_Occurred()) { goto exit; } @@ -241,4 +241,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=805400e4805098ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=431fd0fc40f019d1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 2275b6b9685276..fc6dcaa7651287 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -802,7 +802,7 @@ _codecs_utf_16_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - byteorder = _PyLong_AsInt(args[2]); + byteorder = PyLong_AsInt(args[2]); if (byteorder == -1 && PyErr_Occurred()) { goto exit; } @@ -1091,7 +1091,7 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - byteorder = _PyLong_AsInt(args[2]); + byteorder = PyLong_AsInt(args[2]); if (byteorder == -1 && PyErr_Occurred()) { goto exit; } @@ -1639,7 +1639,7 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("code_page_decode", nargs, 2, 4)) { goto exit; } - codepage = _PyLong_AsInt(args[0]); + codepage = PyLong_AsInt(args[0]); if (codepage == -1 && PyErr_Occurred()) { goto exit; } @@ -1926,7 +1926,7 @@ _codecs_utf_16_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - byteorder = _PyLong_AsInt(args[2]); + byteorder = PyLong_AsInt(args[2]); if (byteorder == -1 && PyErr_Occurred()) { goto exit; } @@ -2099,7 +2099,7 @@ _codecs_utf_32_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - byteorder = _PyLong_AsInt(args[2]); + byteorder = PyLong_AsInt(args[2]); if (byteorder == -1 && PyErr_Occurred()) { goto exit; } @@ -2669,7 +2669,7 @@ _codecs_code_page_encode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("code_page_encode", nargs, 2, 3)) { goto exit; } - code_page = _PyLong_AsInt(args[0]); + code_page = PyLong_AsInt(args[0]); if (code_page == -1 && PyErr_Occurred()) { goto exit; } @@ -2818,4 +2818,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=0f52053d31533376 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d9bc20c8dd36134 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index bb6cc90f0438c0..900058e6d59af0 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -190,11 +190,11 @@ _curses_panel_panel_move(PyCursesPanelObject *self, PyTypeObject *cls, PyObject if (!args) { goto exit; } - y = _PyLong_AsInt(args[0]); + y = PyLong_AsInt(args[0]); if (y == -1 && PyErr_Occurred()) { goto exit; } - x = _PyLong_AsInt(args[1]); + x = PyLong_AsInt(args[1]); if (x == -1 && PyErr_Occurred()) { goto exit; } @@ -422,4 +422,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=8d0533681891523c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cb76cb1d5040f387 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 9d99d41af5d2d9..836087581a73ba 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -708,11 +708,11 @@ _curses_window_enclose(PyCursesWindowObject *self, PyObject *const *args, Py_ssi if (!_PyArg_CheckPositional("enclose", nargs, 2, 2)) { goto exit; } - y = _PyLong_AsInt(args[0]); + y = PyLong_AsInt(args[0]); if (y == -1 && PyErr_Occurred()) { goto exit; } - x = _PyLong_AsInt(args[1]); + x = PyLong_AsInt(args[1]); if (x == -1 && PyErr_Occurred()) { goto exit; } @@ -1264,7 +1264,7 @@ _curses_window_is_linetouched(PyCursesWindowObject *self, PyObject *arg) PyObject *return_value = NULL; int line; - line = _PyLong_AsInt(arg); + line = PyLong_AsInt(arg); if (line == -1 && PyErr_Occurred()) { goto exit; } @@ -1508,11 +1508,11 @@ _curses_window_redrawln(PyCursesWindowObject *self, PyObject *const *args, Py_ss if (!_PyArg_CheckPositional("redrawln", nargs, 2, 2)) { goto exit; } - beg = _PyLong_AsInt(args[0]); + beg = PyLong_AsInt(args[0]); if (beg == -1 && PyErr_Occurred()) { goto exit; } - num = _PyLong_AsInt(args[1]); + num = PyLong_AsInt(args[1]); if (num == -1 && PyErr_Occurred()) { goto exit; } @@ -1607,11 +1607,11 @@ _curses_window_setscrreg(PyCursesWindowObject *self, PyObject *const *args, Py_s if (!_PyArg_CheckPositional("setscrreg", nargs, 2, 2)) { goto exit; } - top = _PyLong_AsInt(args[0]); + top = PyLong_AsInt(args[0]); if (top == -1 && PyErr_Occurred()) { goto exit; } - bottom = _PyLong_AsInt(args[1]); + bottom = PyLong_AsInt(args[1]); if (bottom == -1 && PyErr_Occurred()) { goto exit; } @@ -2009,7 +2009,7 @@ _curses_color_pair(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int pair_number; - pair_number = _PyLong_AsInt(arg); + pair_number = PyLong_AsInt(arg); if (pair_number == -1 && PyErr_Occurred()) { goto exit; } @@ -2045,7 +2045,7 @@ _curses_curs_set(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int visibility; - visibility = _PyLong_AsInt(arg); + visibility = PyLong_AsInt(arg); if (visibility == -1 && PyErr_Occurred()) { goto exit; } @@ -2120,7 +2120,7 @@ _curses_delay_output(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int ms; - ms = _PyLong_AsInt(arg); + ms = PyLong_AsInt(arg); if (ms == -1 && PyErr_Occurred()) { goto exit; } @@ -2363,15 +2363,15 @@ _curses_ungetmouse(PyObject *module, PyObject *const *args, Py_ssize_t nargs) id = (short) ival; } } - x = _PyLong_AsInt(args[1]); + x = PyLong_AsInt(args[1]); if (x == -1 && PyErr_Occurred()) { goto exit; } - y = _PyLong_AsInt(args[2]); + y = PyLong_AsInt(args[2]); if (y == -1 && PyErr_Occurred()) { goto exit; } - z = _PyLong_AsInt(args[3]); + z = PyLong_AsInt(args[3]); if (z == -1 && PyErr_Occurred()) { goto exit; } @@ -2525,7 +2525,7 @@ _curses_has_key(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int key; - key = _PyLong_AsInt(arg); + key = PyLong_AsInt(arg); if (key == -1 && PyErr_Occurred()) { goto exit; } @@ -2744,7 +2744,7 @@ _curses_setupterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO goto skip_optional_pos; } } - fd = _PyLong_AsInt(args[1]); + fd = PyLong_AsInt(args[1]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -2808,7 +2808,7 @@ _curses_set_escdelay(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int ms; - ms = _PyLong_AsInt(arg); + ms = PyLong_AsInt(arg); if (ms == -1 && PyErr_Occurred()) { goto exit; } @@ -2871,7 +2871,7 @@ _curses_set_tabsize(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int size; - size = _PyLong_AsInt(arg); + size = PyLong_AsInt(arg); if (size == -1 && PyErr_Occurred()) { goto exit; } @@ -2957,11 +2957,11 @@ _curses_is_term_resized(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!_PyArg_CheckPositional("is_term_resized", nargs, 2, 2)) { goto exit; } - nlines = _PyLong_AsInt(args[0]); + nlines = PyLong_AsInt(args[0]); if (nlines == -1 && PyErr_Occurred()) { goto exit; } - ncols = _PyLong_AsInt(args[1]); + ncols = PyLong_AsInt(args[1]); if (ncols == -1 && PyErr_Occurred()) { goto exit; } @@ -2994,7 +2994,7 @@ _curses_keyname(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int key; - key = _PyLong_AsInt(arg); + key = PyLong_AsInt(arg); if (key == -1 && PyErr_Occurred()) { goto exit; } @@ -3101,7 +3101,7 @@ _curses_mouseinterval(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int interval; - interval = _PyLong_AsInt(arg); + interval = PyLong_AsInt(arg); if (interval == -1 && PyErr_Occurred()) { goto exit; } @@ -3172,7 +3172,7 @@ _curses_napms(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int ms; - ms = _PyLong_AsInt(arg); + ms = PyLong_AsInt(arg); if (ms == -1 && PyErr_Occurred()) { goto exit; } @@ -3209,11 +3209,11 @@ _curses_newpad(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("newpad", nargs, 2, 2)) { goto exit; } - nlines = _PyLong_AsInt(args[0]); + nlines = PyLong_AsInt(args[0]); if (nlines == -1 && PyErr_Occurred()) { goto exit; } - ncols = _PyLong_AsInt(args[1]); + ncols = PyLong_AsInt(args[1]); if (ncols == -1 && PyErr_Occurred()) { goto exit; } @@ -3471,7 +3471,7 @@ _curses_pair_number(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int attr; - attr = _PyLong_AsInt(arg); + attr = PyLong_AsInt(arg); if (attr == -1 && PyErr_Occurred()) { goto exit; } @@ -3700,11 +3700,11 @@ _curses_resizeterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("resizeterm", nargs, 2, 2)) { goto exit; } - nlines = _PyLong_AsInt(args[0]); + nlines = PyLong_AsInt(args[0]); if (nlines == -1 && PyErr_Occurred()) { goto exit; } - ncols = _PyLong_AsInt(args[1]); + ncols = PyLong_AsInt(args[1]); if (ncols == -1 && PyErr_Occurred()) { goto exit; } @@ -3751,11 +3751,11 @@ _curses_resize_term(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("resize_term", nargs, 2, 2)) { goto exit; } - nlines = _PyLong_AsInt(args[0]); + nlines = PyLong_AsInt(args[0]); if (nlines == -1 && PyErr_Occurred()) { goto exit; } - ncols = _PyLong_AsInt(args[1]); + ncols = PyLong_AsInt(args[1]); if (ncols == -1 && PyErr_Occurred()) { goto exit; } @@ -3816,11 +3816,11 @@ _curses_setsyx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setsyx", nargs, 2, 2)) { goto exit; } - y = _PyLong_AsInt(args[0]); + y = PyLong_AsInt(args[0]); if (y == -1 && PyErr_Occurred()) { goto exit; } - x = _PyLong_AsInt(args[1]); + x = PyLong_AsInt(args[1]); if (x == -1 && PyErr_Occurred()) { goto exit; } @@ -4089,7 +4089,7 @@ _curses_typeahead(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -4313,4 +4313,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=27a2364193b503c1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cff0e570de65d9b8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 51e51e3791cc24..c058c13fcc0683 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -64,15 +64,15 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!fastargs) { goto exit; } - year = _PyLong_AsInt(fastargs[0]); + year = PyLong_AsInt(fastargs[0]); if (year == -1 && PyErr_Occurred()) { goto exit; } - week = _PyLong_AsInt(fastargs[1]); + week = PyLong_AsInt(fastargs[1]); if (week == -1 && PyErr_Occurred()) { goto exit; } - weekday = _PyLong_AsInt(fastargs[2]); + weekday = PyLong_AsInt(fastargs[2]); if (weekday == -1 && PyErr_Occurred()) { goto exit; } @@ -146,4 +146,4 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=42654669940e0e3a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=720c99f24a9aa41c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 98aac07423c8ab..fbf4ccf27c1f71 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -212,7 +212,7 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - mode = _PyLong_AsInt(args[2]); + mode = PyLong_AsInt(args[2]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -222,4 +222,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=b3053c67ecfcc29c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f1bf74536f201669 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 76f6db318f8cc5..73e2d7934213ee 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -334,7 +334,7 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - mode = _PyLong_AsInt(args[2]); + mode = PyLong_AsInt(args[2]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -344,4 +344,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=8c613cbd88e57480 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cf48d9f76fdd62b9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_localemodule.c.h b/Modules/clinic/_localemodule.c.h index 9d65c0876a1a33..18cf5025012e7c 100644 --- a/Modules/clinic/_localemodule.c.h +++ b/Modules/clinic/_localemodule.c.h @@ -30,7 +30,7 @@ _locale_setlocale(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setlocale", nargs, 1, 2)) { goto exit; } - category = _PyLong_AsInt(args[0]); + category = PyLong_AsInt(args[0]); if (category == -1 && PyErr_Occurred()) { goto exit; } @@ -196,7 +196,7 @@ _locale_nl_langinfo(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int item; - item = _PyLong_AsInt(arg); + item = PyLong_AsInt(arg); if (item == -1 && PyErr_Occurred()) { goto exit; } @@ -373,7 +373,7 @@ _locale_dcgettext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - category = _PyLong_AsInt(args[2]); + category = PyLong_AsInt(args[2]); if (category == -1 && PyErr_Occurred()) { goto exit; } @@ -599,4 +599,4 @@ _locale_getencoding(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #define _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #endif /* !defined(_LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF) */ -/*[clinic end generated code: output=9dbd0b4bf5767edd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=119644f17919234d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 9b396a56683921..017b24f765eee3 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -241,7 +241,7 @@ _lzma_LZMADecompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } if (fastargs[0]) { - format = _PyLong_AsInt(fastargs[0]); + format = PyLong_AsInt(fastargs[0]); if (format == -1 && PyErr_Occurred()) { goto exit; } @@ -283,7 +283,7 @@ _lzma_is_check_supported(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int check_id; - check_id = _PyLong_AsInt(arg); + check_id = PyLong_AsInt(arg); if (check_id == -1 && PyErr_Occurred()) { goto exit; } @@ -338,4 +338,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=96c1fbdada1ef232 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aaf225a5d15d3e75 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 52fbdcbd2a6528..f8e6a0a54f940c 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -61,7 +61,7 @@ _opcode_stack_effect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -135,7 +135,7 @@ _opcode_is_valid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -198,7 +198,7 @@ _opcode_has_arg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -261,7 +261,7 @@ _opcode_has_const(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -324,7 +324,7 @@ _opcode_has_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -387,7 +387,7 @@ _opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -455,7 +455,7 @@ _opcode_has_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -518,7 +518,7 @@ _opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -581,7 +581,7 @@ _opcode_has_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!args) { goto exit; } - opcode = _PyLong_AsInt(args[0]); + opcode = PyLong_AsInt(args[0]); if (opcode == -1 && PyErr_Occurred()) { goto exit; } @@ -668,4 +668,4 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=31a1a11c2f81dca4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ce89aee80dd825d2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index f08878cf668908..3b19a47e716492 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -98,35 +98,35 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) py_fds_to_keep = args[3]; cwd_obj = args[4]; env_list = args[5]; - p2cread = _PyLong_AsInt(args[6]); + p2cread = PyLong_AsInt(args[6]); if (p2cread == -1 && PyErr_Occurred()) { goto exit; } - p2cwrite = _PyLong_AsInt(args[7]); + p2cwrite = PyLong_AsInt(args[7]); if (p2cwrite == -1 && PyErr_Occurred()) { goto exit; } - c2pread = _PyLong_AsInt(args[8]); + c2pread = PyLong_AsInt(args[8]); if (c2pread == -1 && PyErr_Occurred()) { goto exit; } - c2pwrite = _PyLong_AsInt(args[9]); + c2pwrite = PyLong_AsInt(args[9]); if (c2pwrite == -1 && PyErr_Occurred()) { goto exit; } - errread = _PyLong_AsInt(args[10]); + errread = PyLong_AsInt(args[10]); if (errread == -1 && PyErr_Occurred()) { goto exit; } - errwrite = _PyLong_AsInt(args[11]); + errwrite = PyLong_AsInt(args[11]); if (errwrite == -1 && PyErr_Occurred()) { goto exit; } - errpipe_read = _PyLong_AsInt(args[12]); + errpipe_read = PyLong_AsInt(args[12]); if (errpipe_read == -1 && PyErr_Occurred()) { goto exit; } - errpipe_write = _PyLong_AsInt(args[13]); + errpipe_write = PyLong_AsInt(args[13]); if (errpipe_write == -1 && PyErr_Occurred()) { goto exit; } @@ -145,7 +145,7 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) gid_object = args[17]; extra_groups_packed = args[18]; uid_object = args[19]; - child_umask = _PyLong_AsInt(args[20]); + child_umask = PyLong_AsInt(args[20]); if (child_umask == -1 && PyErr_Occurred()) { goto exit; } @@ -159,4 +159,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=46d71e86845c93d7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=be0e9b5d8c0217fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index ec8531ce006649..45f06b60ed4448 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -106,7 +106,7 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) PyObject *return_value = NULL; int k; - k = _PyLong_AsInt(arg); + k = PyLong_AsInt(arg); if (k == -1 && PyErr_Occurred()) { goto exit; } @@ -115,4 +115,4 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=bc17406a886824fc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3feabc783b317d0d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 9f967ddc8e3061..81712617bee12b 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -445,7 +445,7 @@ _ssl__SSLContext(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!_PyArg_CheckPositional("_SSLContext", PyTuple_GET_SIZE(args), 1, 1)) { goto exit; } - proto_version = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); + proto_version = PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); if (proto_version == -1 && PyErr_Occurred()) { goto exit; } @@ -1073,7 +1073,7 @@ _ssl_MemoryBIO_read(PySSLMemoryBIO *self, PyObject *const *args, Py_ssize_t narg if (nargs < 1) { goto skip_optional; } - len = _PyLong_AsInt(args[0]); + len = PyLong_AsInt(args[0]); if (len == -1 && PyErr_Occurred()) { goto exit; } @@ -1223,7 +1223,7 @@ _ssl_RAND_bytes(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int n; - n = _PyLong_AsInt(arg); + n = PyLong_AsInt(arg); if (n == -1 && PyErr_Occurred()) { goto exit; } @@ -1372,7 +1372,7 @@ _ssl_nid2obj(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int nid; - nid = _PyLong_AsInt(arg); + nid = PyLong_AsInt(arg); if (nid == -1 && PyErr_Occurred()) { goto exit; } @@ -1542,4 +1542,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=4d9b81fa81f520f0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4c9f00c62f0825d2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 51c3a7824dbebd..d3a064605f0e41 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -181,7 +181,7 @@ bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - c = _PyLong_AsInt(args[2]); + c = PyLong_AsInt(args[2]); if (c == -1 && PyErr_Occurred()) { goto exit; } @@ -627,14 +627,14 @@ int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - a = _PyLong_AsInt(args[0]); + a = PyLong_AsInt(args[0]); if (a == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 2) { goto skip_optional; } - b = _PyLong_AsInt(args[1]); + b = PyLong_AsInt(args[1]); if (b == -1 && PyErr_Occurred()) { goto exit; } @@ -3069,4 +3069,4 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=0de394419fefe7cf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=86396cbed6eb8b65 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 9419dcd751a0e9..72a5f166099659 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -123,14 +123,14 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s } ast = args[0]; filename = args[1]; - optimize = _PyLong_AsInt(args[2]); + optimize = PyLong_AsInt(args[2]); if (optimize == -1 && PyErr_Occurred()) { goto exit; } if (!noptargs) { goto skip_optional_pos; } - compile_mode = _PyLong_AsInt(args[3]); + compile_mode = PyLong_AsInt(args[3]); if (compile_mode == -1 && PyErr_Occurred()) { goto exit; } @@ -194,7 +194,7 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize } instructions = args[0]; consts = args[1]; - nlocals = _PyLong_AsInt(args[2]); + nlocals = PyLong_AsInt(args[2]); if (nlocals == -1 && PyErr_Occurred()) { goto exit; } @@ -265,4 +265,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=811d50772c8f285a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ce6e2257e95a853 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 42ec7475e5e4be..f7b99b1c52f0a9 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -117,7 +117,7 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se goto skip_optional_pos; } if (args[0]) { - n = _PyLong_AsInt(args[0]); + n = PyLong_AsInt(args[0]); if (n == -1 && PyErr_Occurred()) { goto exit; } @@ -162,4 +162,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=52ea97ab2f03bb6d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6f8db3983c5312a0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 96c6ee26f426c3..e588ea6989c672 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -432,7 +432,7 @@ _tkinter_tkapp_createfilehandler(TkappObject *self, PyObject *const *args, Py_ss goto exit; } file = args[0]; - mask = _PyLong_AsInt(args[1]); + mask = PyLong_AsInt(args[1]); if (mask == -1 && PyErr_Occurred()) { goto exit; } @@ -496,7 +496,7 @@ _tkinter_tkapp_createtimerhandler(TkappObject *self, PyObject *const *args, Py_s if (!_PyArg_CheckPositional("createtimerhandler", nargs, 2, 2)) { goto exit; } - milliseconds = _PyLong_AsInt(args[0]); + milliseconds = PyLong_AsInt(args[0]); if (milliseconds == -1 && PyErr_Occurred()) { goto exit; } @@ -530,7 +530,7 @@ _tkinter_tkapp_mainloop(TkappObject *self, PyObject *const *args, Py_ssize_t nar if (nargs < 1) { goto skip_optional; } - threshold = _PyLong_AsInt(args[0]); + threshold = PyLong_AsInt(args[0]); if (threshold == -1 && PyErr_Occurred()) { goto exit; } @@ -564,7 +564,7 @@ _tkinter_tkapp_dooneevent(TkappObject *self, PyObject *const *args, Py_ssize_t n if (nargs < 1) { goto skip_optional; } - flags = _PyLong_AsInt(args[0]); + flags = PyLong_AsInt(args[0]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -820,7 +820,7 @@ _tkinter_setbusywaitinterval(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int new_val; - new_val = _PyLong_AsInt(arg); + new_val = PyLong_AsInt(arg); if (new_val == -1 && PyErr_Occurred()) { goto exit; } @@ -865,4 +865,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=2a4e3bf8448604b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2cdb2fd0ce66ebe5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tracemalloc.c.h b/Modules/clinic/_tracemalloc.c.h index a89cd9aabca8d6..aa4fe84939aaa2 100644 --- a/Modules/clinic/_tracemalloc.c.h +++ b/Modules/clinic/_tracemalloc.c.h @@ -107,7 +107,7 @@ _tracemalloc_start(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - nframe = _PyLong_AsInt(args[0]); + nframe = PyLong_AsInt(args[0]); if (nframe == -1 && PyErr_Occurred()) { goto exit; } @@ -218,4 +218,4 @@ _tracemalloc_reset_peak(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _tracemalloc_reset_peak_impl(module); } -/*[clinic end generated code: output=44e3f8553aae2535 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=98a42e95af7eaf09 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 600931835ecac1..5e957013ad0dc3 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -589,7 +589,7 @@ array__array_reconstructor(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } typecode = PyUnicode_READ_CHAR(args[1], 0); - mformat_code = _PyLong_AsInt(args[2]); + mformat_code = PyLong_AsInt(args[2]); if (mformat_code == -1 && PyErr_Occurred()) { goto exit; } @@ -674,4 +674,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=11595e9f38d6d500 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db417f5677a25eaa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 63566dfb10e74f..8af49b4d87afc2 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -448,7 +448,7 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } } - bytes_per_sep = _PyLong_AsInt(args[2]); + bytes_per_sep = PyLong_AsInt(args[2]); if (bytes_per_sep == -1 && PyErr_Occurred()) { goto exit; } @@ -541,7 +541,7 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } } - bytes_per_sep = _PyLong_AsInt(args[2]); + bytes_per_sep = PyLong_AsInt(args[2]); if (bytes_per_sep == -1 && PyErr_Occurred()) { goto exit; } @@ -795,4 +795,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=ab156917c9db79d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf950ef45a10928d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 20eb50b0e76b38..bd978b63e4dd6b 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -44,7 +44,7 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - code = _PyLong_AsInt(args[1]); + code = PyLong_AsInt(args[1]); if (code == -1 && PyErr_Occurred()) { goto exit; } @@ -164,7 +164,7 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - code = _PyLong_AsInt(args[1]); + code = PyLong_AsInt(args[1]); if (code == -1 && PyErr_Occurred()) { goto exit; } @@ -224,7 +224,7 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - code = _PyLong_AsInt(args[1]); + code = PyLong_AsInt(args[1]); if (code == -1 && PyErr_Occurred()) { goto exit; } @@ -239,7 +239,7 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - whence = _PyLong_AsInt(args[4]); + whence = PyLong_AsInt(args[4]); if (whence == -1 && PyErr_Occurred()) { goto exit; } @@ -249,4 +249,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=1db859412172dd53 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=705976d5f53f2272 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 2d18e2ee0978e0..0ea5b40cfb770a 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -131,7 +131,7 @@ gc_collect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (!noptargs) { goto skip_optional_pos; } - generation = _PyLong_AsInt(args[0]); + generation = PyLong_AsInt(args[0]); if (generation == -1 && PyErr_Occurred()) { goto exit; } @@ -175,7 +175,7 @@ gc_set_debug(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flags; - flags = _PyLong_AsInt(arg); + flags = PyLong_AsInt(arg); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -424,4 +424,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=66432ac0e17fd04f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=16003cfbc66ada39 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 89928312211b60..b6c5f79acb1f6b 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -288,11 +288,11 @@ _overlapped_CreateEvent(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } EventAttributes = args[0]; - ManualReset = _PyLong_AsInt(args[1]); + ManualReset = PyLong_AsInt(args[1]); if (ManualReset == -1 && PyErr_Occurred()) { goto exit; } - InitialState = _PyLong_AsInt(args[2]); + InitialState = PyLong_AsInt(args[2]); if (InitialState == -1 && PyErr_Occurred()) { goto exit; } @@ -402,7 +402,7 @@ _overlapped_BindLocal(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!Socket && PyErr_Occurred()) { goto exit; } - Family = _PyLong_AsInt(args[1]); + Family = PyLong_AsInt(args[1]); if (Family == -1 && PyErr_Occurred()) { goto exit; } @@ -546,7 +546,7 @@ _overlapped_Overlapped_getresult(OverlappedObject *self, PyObject *const *args, if (nargs < 1) { goto skip_optional; } - wait = _PyLong_AsInt(args[0]); + wait = PyLong_AsInt(args[0]); if (wait == -1 && PyErr_Occurred()) { goto exit; } @@ -1262,4 +1262,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=05fd038b8a81272d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9fbc01f706562dea input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9b3fa8ded26766..db82916b901f4b 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -262,7 +262,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!path_converter(args[0], &path)) { goto exit; } - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -327,7 +327,7 @@ os_ttyname(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -568,7 +568,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!path_converter(args[0], &path)) { goto exit; } - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -658,11 +658,11 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!args) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -731,7 +731,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!path_converter(args[0], &path)) { goto exit; } - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -1284,7 +1284,7 @@ os_fchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!args) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -2329,7 +2329,7 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw goto skip_optional_pos; } if (args[1]) { - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -2374,7 +2374,7 @@ os_nice(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int increment; - increment = _PyLong_AsInt(arg); + increment = PyLong_AsInt(arg); if (increment == -1 && PyErr_Occurred()) { goto exit; } @@ -2437,11 +2437,11 @@ os_getpriority(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje if (!args) { goto exit; } - which = _PyLong_AsInt(args[0]); + which = PyLong_AsInt(args[0]); if (which == -1 && PyErr_Occurred()) { goto exit; } - who = _PyLong_AsInt(args[1]); + who = PyLong_AsInt(args[1]); if (who == -1 && PyErr_Occurred()) { goto exit; } @@ -2505,15 +2505,15 @@ os_setpriority(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje if (!args) { goto exit; } - which = _PyLong_AsInt(args[0]); + which = PyLong_AsInt(args[0]); if (which == -1 && PyErr_Occurred()) { goto exit; } - who = _PyLong_AsInt(args[1]); + who = PyLong_AsInt(args[1]); if (who == -1 && PyErr_Occurred()) { goto exit; } - priority = _PyLong_AsInt(args[2]); + priority = PyLong_AsInt(args[2]); if (priority == -1 && PyErr_Occurred()) { goto exit; } @@ -2944,7 +2944,7 @@ os_umask(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int mask; - mask = _PyLong_AsInt(arg); + mask = PyLong_AsInt(arg); if (mask == -1 && PyErr_Occurred()) { goto exit; } @@ -3298,7 +3298,7 @@ os__exit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -3759,7 +3759,7 @@ os_spawnv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("spawnv", nargs, 3, 3)) { goto exit; } - mode = _PyLong_AsInt(args[0]); + mode = PyLong_AsInt(args[0]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -3814,7 +3814,7 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("spawnve", nargs, 4, 4)) { goto exit; } - mode = _PyLong_AsInt(args[0]); + mode = PyLong_AsInt(args[0]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -4023,7 +4023,7 @@ os_sched_get_priority_max(PyObject *module, PyObject *const *args, Py_ssize_t na if (!args) { goto exit; } - policy = _PyLong_AsInt(args[0]); + policy = PyLong_AsInt(args[0]); if (policy == -1 && PyErr_Occurred()) { goto exit; } @@ -4085,7 +4085,7 @@ os_sched_get_priority_min(PyObject *module, PyObject *const *args, Py_ssize_t na if (!args) { goto exit; } - policy = _PyLong_AsInt(args[0]); + policy = PyLong_AsInt(args[0]); if (policy == -1 && PyErr_Occurred()) { goto exit; } @@ -4644,7 +4644,7 @@ os_getgrouplist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - basegid = _PyLong_AsInt(args[1]); + basegid = PyLong_AsInt(args[1]); if (basegid == -1 && PyErr_Occurred()) { goto exit; } @@ -4762,7 +4762,7 @@ os_initgroups(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!PyUnicode_FSConverter(args[0], &oname)) { goto exit; } - gid = _PyLong_AsInt(args[1]); + gid = PyLong_AsInt(args[1]); if (gid == -1 && PyErr_Occurred()) { goto exit; } @@ -5078,7 +5078,7 @@ os_plock(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int op; - op = _PyLong_AsInt(arg); + op = PyLong_AsInt(arg); if (op == -1 && PyErr_Occurred()) { goto exit; } @@ -5356,7 +5356,7 @@ os_wait3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - options = _PyLong_AsInt(args[0]); + options = PyLong_AsInt(args[0]); if (options == -1 && PyErr_Occurred()) { goto exit; } @@ -5699,7 +5699,7 @@ os_setns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!noptargs) { goto skip_optional_pos; } - nstype = _PyLong_AsInt(args[1]); + nstype = PyLong_AsInt(args[1]); if (nstype == -1 && PyErr_Occurred()) { goto exit; } @@ -5765,7 +5765,7 @@ os_unshare(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (!args) { goto exit; } - flags = _PyLong_AsInt(args[0]); + flags = PyLong_AsInt(args[0]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -6092,7 +6092,7 @@ os_tcgetpgrp(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6198,7 +6198,7 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn if (!path_converter(args[0], &path)) { goto exit; } - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -6206,7 +6206,7 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn goto skip_optional_pos; } if (args[2]) { - mode = _PyLong_AsInt(args[2]); + mode = PyLong_AsInt(args[2]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -6283,7 +6283,7 @@ os_close(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6315,11 +6315,11 @@ os_closerange(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("closerange", nargs, 2, 2)) { goto exit; } - fd_low = _PyLong_AsInt(args[0]); + fd_low = PyLong_AsInt(args[0]); if (fd_low == -1 && PyErr_Occurred()) { goto exit; } - fd_high = _PyLong_AsInt(args[1]); + fd_high = PyLong_AsInt(args[1]); if (fd_high == -1 && PyErr_Occurred()) { goto exit; } @@ -6348,7 +6348,7 @@ os_dup(PyObject *module, PyObject *arg) int fd; int _return_value; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6416,11 +6416,11 @@ os_dup2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn if (!args) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } - fd2 = _PyLong_AsInt(args[1]); + fd2 = PyLong_AsInt(args[1]); if (fd2 == -1 && PyErr_Occurred()) { goto exit; } @@ -6476,11 +6476,11 @@ os_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("lockf", nargs, 3, 3)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } - command = _PyLong_AsInt(args[1]); + command = PyLong_AsInt(args[1]); if (command == -1 && PyErr_Occurred()) { goto exit; } @@ -6531,14 +6531,14 @@ os_lseek(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("lseek", nargs, 3, 3)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } if (!Py_off_t_converter(args[1], &position)) { goto exit; } - how = _PyLong_AsInt(args[2]); + how = PyLong_AsInt(args[2]); if (how == -1 && PyErr_Occurred()) { goto exit; } @@ -6574,7 +6574,7 @@ os_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("read", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6629,7 +6629,7 @@ os_readv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("readv", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6674,7 +6674,7 @@ os_pread(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("pread", nargs, 3, 3)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6743,7 +6743,7 @@ os_preadv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("preadv", nargs, 3, 4)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6754,7 +6754,7 @@ os_preadv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - flags = _PyLong_AsInt(args[3]); + flags = PyLong_AsInt(args[3]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -6794,7 +6794,7 @@ os_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("write", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6880,11 +6880,11 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - out_fd = _PyLong_AsInt(args[0]); + out_fd = PyLong_AsInt(args[0]); if (out_fd == -1 && PyErr_Occurred()) { goto exit; } - in_fd = _PyLong_AsInt(args[1]); + in_fd = PyLong_AsInt(args[1]); if (in_fd == -1 && PyErr_Occurred()) { goto exit; } @@ -6909,7 +6909,7 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } } - flags = _PyLong_AsInt(args[6]); + flags = PyLong_AsInt(args[6]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -6982,11 +6982,11 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - out_fd = _PyLong_AsInt(args[0]); + out_fd = PyLong_AsInt(args[0]); if (out_fd == -1 && PyErr_Occurred()) { goto exit; } - in_fd = _PyLong_AsInt(args[1]); + in_fd = PyLong_AsInt(args[1]); if (in_fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7020,7 +7020,7 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } } - flags = _PyLong_AsInt(args[6]); + flags = PyLong_AsInt(args[6]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -7087,11 +7087,11 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - out_fd = _PyLong_AsInt(args[0]); + out_fd = PyLong_AsInt(args[0]); if (out_fd == -1 && PyErr_Occurred()) { goto exit; } - in_fd = _PyLong_AsInt(args[1]); + in_fd = PyLong_AsInt(args[1]); if (in_fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7141,15 +7141,15 @@ os__fcopyfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("_fcopyfile", nargs, 3, 3)) { goto exit; } - in_fd = _PyLong_AsInt(args[0]); + in_fd = PyLong_AsInt(args[0]); if (in_fd == -1 && PyErr_Occurred()) { goto exit; } - out_fd = _PyLong_AsInt(args[1]); + out_fd = PyLong_AsInt(args[1]); if (out_fd == -1 && PyErr_Occurred()) { goto exit; } - flags = _PyLong_AsInt(args[2]); + flags = PyLong_AsInt(args[2]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -7212,7 +7212,7 @@ os_fstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7244,7 +7244,7 @@ os_isatty(PyObject *module, PyObject *arg) int fd; int _return_value; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7309,7 +7309,7 @@ os_pipe2(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flags; - flags = _PyLong_AsInt(arg); + flags = PyLong_AsInt(arg); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -7349,7 +7349,7 @@ os_writev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("writev", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7396,7 +7396,7 @@ os_pwrite(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("pwrite", nargs, 3, 3)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7470,7 +7470,7 @@ os_pwritev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("pwritev", nargs, 3, 4)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -7481,7 +7481,7 @@ os_pwritev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - flags = _PyLong_AsInt(args[3]); + flags = PyLong_AsInt(args[3]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -7569,11 +7569,11 @@ os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py if (!args) { goto exit; } - src = _PyLong_AsInt(args[0]); + src = PyLong_AsInt(args[0]); if (src == -1 && PyErr_Occurred()) { goto exit; } - dst = _PyLong_AsInt(args[1]); + dst = PyLong_AsInt(args[1]); if (dst == -1 && PyErr_Occurred()) { goto exit; } @@ -7684,11 +7684,11 @@ os_splice(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!args) { goto exit; } - src = _PyLong_AsInt(args[0]); + src = PyLong_AsInt(args[0]); if (src == -1 && PyErr_Occurred()) { goto exit; } - dst = _PyLong_AsInt(args[1]); + dst = PyLong_AsInt(args[1]); if (dst == -1 && PyErr_Occurred()) { goto exit; } @@ -7796,7 +7796,7 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k goto skip_optional_pos; } if (args[1]) { - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -7897,7 +7897,7 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw goto skip_optional_pos; } if (args[1]) { - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -8029,11 +8029,11 @@ os_makedev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("makedev", nargs, 2, 2)) { goto exit; } - major = _PyLong_AsInt(args[0]); + major = PyLong_AsInt(args[0]); if (major == -1 && PyErr_Occurred()) { goto exit; } - minor = _PyLong_AsInt(args[1]); + minor = PyLong_AsInt(args[1]); if (minor == -1 && PyErr_Occurred()) { goto exit; } @@ -8073,7 +8073,7 @@ os_ftruncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("ftruncate", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -8188,7 +8188,7 @@ os_posix_fallocate(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("posix_fallocate", nargs, 3, 3)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -8241,7 +8241,7 @@ os_posix_fadvise(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("posix_fadvise", nargs, 4, 4)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -8251,7 +8251,7 @@ os_posix_fadvise(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!Py_off_t_converter(args[2], &length)) { goto exit; } - advice = _PyLong_AsInt(args[3]); + advice = PyLong_AsInt(args[3]); if (advice == -1 && PyErr_Occurred()) { goto exit; } @@ -8433,7 +8433,7 @@ os_strerror(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int code; - code = _PyLong_AsInt(arg); + code = PyLong_AsInt(arg); if (code == -1 && PyErr_Occurred()) { goto exit; } @@ -8464,7 +8464,7 @@ os_WCOREDUMP(PyObject *module, PyObject *arg) int status; int _return_value; - status = _PyLong_AsInt(arg); + status = PyLong_AsInt(arg); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8534,7 +8534,7 @@ os_WIFCONTINUED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8601,7 +8601,7 @@ os_WIFSTOPPED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8668,7 +8668,7 @@ os_WIFSIGNALED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8735,7 +8735,7 @@ os_WIFEXITED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8802,7 +8802,7 @@ os_WEXITSTATUS(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8869,7 +8869,7 @@ os_WTERMSIG(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8936,7 +8936,7 @@ os_WSTOPSIG(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - status = _PyLong_AsInt(args[0]); + status = PyLong_AsInt(args[0]); if (status == -1 && PyErr_Occurred()) { goto exit; } @@ -8974,7 +8974,7 @@ os_fstatvfs(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -9451,7 +9451,7 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } } - show_cmd = _PyLong_AsInt(args[4]); + show_cmd = PyLong_AsInt(args[4]); if (show_cmd == -1 && PyErr_Occurred()) { goto exit; } @@ -9550,7 +9550,7 @@ os_device_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py if (!args) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -9853,7 +9853,7 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[3]) { - flags = _PyLong_AsInt(args[3]); + flags = PyLong_AsInt(args[3]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -10229,7 +10229,7 @@ os_eventfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (!noptargs) { goto skip_optional_pos; } - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -10406,7 +10406,7 @@ os_get_terminal_size(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -10460,7 +10460,7 @@ os_get_inheritable(PyObject *module, PyObject *arg) int fd; int _return_value; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -10496,11 +10496,11 @@ os_set_inheritable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("set_inheritable", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } - inheritable = _PyLong_AsInt(args[1]); + inheritable = PyLong_AsInt(args[1]); if (inheritable == -1 && PyErr_Occurred()) { goto exit; } @@ -10601,7 +10601,7 @@ os_get_blocking(PyObject *module, PyObject *arg) int fd; int _return_value; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -10640,7 +10640,7 @@ os_set_blocking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("set_blocking", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -11153,7 +11153,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_pos; } - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -11990,4 +11990,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9a5f78bb65470528 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e839ce21678ea66 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 34937c5d594f5c..97247781eaa072 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -288,7 +288,7 @@ pyexpat_xmlparser_SetParamEntityParsing(xmlparseobject *self, PyObject *arg) PyObject *return_value = NULL; int flag; - flag = _PyLong_AsInt(arg); + flag = PyLong_AsInt(arg); if (flag == -1 && PyErr_Occurred()) { goto exit; } @@ -498,4 +498,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=63efc62e24a7b5a7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b30852bfc778208 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index 83d93ef7e7b4f1..0bd6e36851e45d 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -146,7 +146,7 @@ readline_append_history_file(PyObject *module, PyObject *const *args, Py_ssize_t if (!_PyArg_CheckPositional("append_history_file", nargs, 1, 2)) { goto exit; } - nelements = _PyLong_AsInt(args[0]); + nelements = PyLong_AsInt(args[0]); if (nelements == -1 && PyErr_Occurred()) { goto exit; } @@ -183,7 +183,7 @@ readline_set_history_length(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int length; - length = _PyLong_AsInt(arg); + length = PyLong_AsInt(arg); if (length == -1 && PyErr_Occurred()) { goto exit; } @@ -404,7 +404,7 @@ readline_remove_history_item(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int entry_number; - entry_number = _PyLong_AsInt(arg); + entry_number = PyLong_AsInt(arg); if (entry_number == -1 && PyErr_Occurred()) { goto exit; } @@ -439,7 +439,7 @@ readline_replace_history_item(PyObject *module, PyObject *const *args, Py_ssize_ if (!_PyArg_CheckPositional("replace_history_item", nargs, 2, 2)) { goto exit; } - entry_number = _PyLong_AsInt(args[0]); + entry_number = PyLong_AsInt(args[0]); if (entry_number == -1 && PyErr_Occurred()) { goto exit; } @@ -582,7 +582,7 @@ readline_get_history_item(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int idx; - idx = _PyLong_AsInt(arg); + idx = PyLong_AsInt(arg); if (idx == -1 && PyErr_Occurred()) { goto exit; } @@ -688,4 +688,4 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=d66c3df7e72f93c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c9c5709b3e00c8b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index d0ca8e7150fa86..d769a7f612357d 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -27,7 +27,7 @@ resource_getrusage(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int who; - who = _PyLong_AsInt(arg); + who = PyLong_AsInt(arg); if (who == -1 && PyErr_Occurred()) { goto exit; } @@ -56,7 +56,7 @@ resource_getrlimit(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int resource; - resource = _PyLong_AsInt(arg); + resource = PyLong_AsInt(arg); if (resource == -1 && PyErr_Occurred()) { goto exit; } @@ -87,7 +87,7 @@ resource_setrlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setrlimit", nargs, 2, 2)) { goto exit; } - resource = _PyLong_AsInt(args[0]); + resource = PyLong_AsInt(args[0]); if (resource == -1 && PyErr_Occurred()) { goto exit; } @@ -127,7 +127,7 @@ resource_prlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (pid == -1 && PyErr_Occurred()) { goto exit; } - resource = _PyLong_AsInt(args[1]); + resource = PyLong_AsInt(args[1]); if (resource == -1 && PyErr_Occurred()) { goto exit; } @@ -178,4 +178,4 @@ resource_getpagesize(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=2fbec74335a57230 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=916ecf0eb1927c37 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index f44ca1d70a1e86..bebcc24d67e3b8 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -568,7 +568,7 @@ select_epoll(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } if (fastargs[0]) { - sizehint = _PyLong_AsInt(fastargs[0]); + sizehint = PyLong_AsInt(fastargs[0]); if (sizehint == -1 && PyErr_Occurred()) { goto exit; } @@ -576,7 +576,7 @@ select_epoll(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - flags = _PyLong_AsInt(fastargs[1]); + flags = PyLong_AsInt(fastargs[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -655,7 +655,7 @@ select_epoll_fromfd(PyTypeObject *type, PyObject *arg) PyObject *return_value = NULL; int fd; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -954,7 +954,7 @@ select_epoll_poll(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } } - maxevents = _PyLong_AsInt(args[1]); + maxevents = PyLong_AsInt(args[1]); if (maxevents == -1 && PyErr_Occurred()) { goto exit; } @@ -1145,7 +1145,7 @@ select_kqueue_fromfd(PyTypeObject *type, PyObject *arg) PyObject *return_value = NULL; int fd; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -1193,7 +1193,7 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize goto exit; } changelist = args[0]; - maxevents = _PyLong_AsInt(args[1]); + maxevents = PyLong_AsInt(args[1]); if (maxevents == -1 && PyErr_Occurred()) { goto exit; } @@ -1309,4 +1309,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=64516114287e894d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7521d757ef9e63e8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 3b3c6ba150a1ad..f0ae699a3f1e18 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -33,7 +33,7 @@ signal_default_int_handler(PyObject *module, PyObject *const *args, Py_ssize_t n if (!_PyArg_CheckPositional("default_int_handler", nargs, 2, 2)) { goto exit; } - signalnum = _PyLong_AsInt(args[0]); + signalnum = PyLong_AsInt(args[0]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -65,7 +65,7 @@ signal_alarm(PyObject *module, PyObject *arg) int seconds; long _return_value; - seconds = _PyLong_AsInt(arg); + seconds = PyLong_AsInt(arg); if (seconds == -1 && PyErr_Occurred()) { goto exit; } @@ -121,7 +121,7 @@ signal_raise_signal(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int signalnum; - signalnum = _PyLong_AsInt(arg); + signalnum = PyLong_AsInt(arg); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -160,7 +160,7 @@ signal_signal(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("signal", nargs, 2, 2)) { goto exit; } - signalnum = _PyLong_AsInt(args[0]); + signalnum = PyLong_AsInt(args[0]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -195,7 +195,7 @@ signal_getsignal(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int signalnum; - signalnum = _PyLong_AsInt(arg); + signalnum = PyLong_AsInt(arg); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -227,7 +227,7 @@ signal_strsignal(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int signalnum; - signalnum = _PyLong_AsInt(arg); + signalnum = PyLong_AsInt(arg); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -264,11 +264,11 @@ signal_siginterrupt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("siginterrupt", nargs, 2, 2)) { goto exit; } - signalnum = _PyLong_AsInt(args[0]); + signalnum = PyLong_AsInt(args[0]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } - flag = _PyLong_AsInt(args[1]); + flag = PyLong_AsInt(args[1]); if (flag == -1 && PyErr_Occurred()) { goto exit; } @@ -311,7 +311,7 @@ signal_setitimer(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setitimer", nargs, 2, 3)) { goto exit; } - which = _PyLong_AsInt(args[0]); + which = PyLong_AsInt(args[0]); if (which == -1 && PyErr_Occurred()) { goto exit; } @@ -349,7 +349,7 @@ signal_getitimer(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int which; - which = _PyLong_AsInt(arg); + which = PyLong_AsInt(arg); if (which == -1 && PyErr_Occurred()) { goto exit; } @@ -385,7 +385,7 @@ signal_pthread_sigmask(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (!_PyArg_CheckPositional("pthread_sigmask", nargs, 2, 2)) { goto exit; } - how = _PyLong_AsInt(args[0]); + how = PyLong_AsInt(args[0]); if (how == -1 && PyErr_Occurred()) { goto exit; } @@ -587,7 +587,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } thread_id = PyLong_AsUnsignedLongMask(args[0]); - signalnum = _PyLong_AsInt(args[1]); + signalnum = PyLong_AsInt(args[1]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -626,11 +626,11 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("pidfd_send_signal", nargs, 2, 4)) { goto exit; } - pidfd = _PyLong_AsInt(args[0]); + pidfd = PyLong_AsInt(args[0]); if (pidfd == -1 && PyErr_Occurred()) { goto exit; } - signalnum = _PyLong_AsInt(args[1]); + signalnum = PyLong_AsInt(args[1]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; } @@ -641,7 +641,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - flags = _PyLong_AsInt(args[3]); + flags = PyLong_AsInt(args[3]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -705,4 +705,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=2b54dc607f6e3146 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9903be3eba6cbc1c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 8ff1044d013b0f..5e2f581c3e0600 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -58,7 +58,7 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } if (fastargs[0]) { - family = _PyLong_AsInt(fastargs[0]); + family = PyLong_AsInt(fastargs[0]); if (family == -1 && PyErr_Occurred()) { goto exit; } @@ -67,7 +67,7 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[1]) { - type = _PyLong_AsInt(fastargs[1]); + type = PyLong_AsInt(fastargs[1]); if (type == -1 && PyErr_Occurred()) { goto exit; } @@ -76,7 +76,7 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - proto = _PyLong_AsInt(fastargs[2]); + proto = PyLong_AsInt(fastargs[2]); if (proto == -1 && PyErr_Occurred()) { goto exit; } @@ -91,4 +91,4 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=987155ac4b48a198 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c44e3ac23999ac60 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 78863e53c42ffb..4e911aa0a5fb9b 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -75,7 +75,7 @@ termios_tcsetattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - when = _PyLong_AsInt(args[1]); + when = PyLong_AsInt(args[1]); if (when == -1 && PyErr_Occurred()) { goto exit; } @@ -114,7 +114,7 @@ termios_tcsendbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - duration = _PyLong_AsInt(args[1]); + duration = PyLong_AsInt(args[1]); if (duration == -1 && PyErr_Occurred()) { goto exit; } @@ -180,7 +180,7 @@ termios_tcflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - queue = _PyLong_AsInt(args[1]); + queue = PyLong_AsInt(args[1]); if (queue == -1 && PyErr_Occurred()) { goto exit; } @@ -219,7 +219,7 @@ termios_tcflow(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { goto exit; } - action = _PyLong_AsInt(args[1]); + action = PyLong_AsInt(args[1]); if (action == -1 && PyErr_Occurred()) { goto exit; } @@ -292,4 +292,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d286a3906a051869 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4c79a3bf87370275 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 65412b2435ade1..3d71536c846109 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -77,7 +77,7 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec goto skip_optional_pos; } if (args[1]) { - level = _PyLong_AsInt(args[1]); + level = PyLong_AsInt(args[1]); if (level == -1 && PyErr_Occurred()) { goto exit; } @@ -85,7 +85,7 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec goto skip_optional_pos; } } - wbits = _PyLong_AsInt(args[2]); + wbits = PyLong_AsInt(args[2]); if (wbits == -1 && PyErr_Occurred()) { goto exit; } @@ -171,7 +171,7 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto skip_optional_pos; } if (args[1]) { - wbits = _PyLong_AsInt(args[1]); + wbits = PyLong_AsInt(args[1]); if (wbits == -1 && PyErr_Occurred()) { goto exit; } @@ -286,7 +286,7 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } if (args[0]) { - level = _PyLong_AsInt(args[0]); + level = PyLong_AsInt(args[0]); if (level == -1 && PyErr_Occurred()) { goto exit; } @@ -295,7 +295,7 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb } } if (args[1]) { - method = _PyLong_AsInt(args[1]); + method = PyLong_AsInt(args[1]); if (method == -1 && PyErr_Occurred()) { goto exit; } @@ -304,7 +304,7 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb } } if (args[2]) { - wbits = _PyLong_AsInt(args[2]); + wbits = PyLong_AsInt(args[2]); if (wbits == -1 && PyErr_Occurred()) { goto exit; } @@ -313,7 +313,7 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb } } if (args[3]) { - memLevel = _PyLong_AsInt(args[3]); + memLevel = PyLong_AsInt(args[3]); if (memLevel == -1 && PyErr_Occurred()) { goto exit; } @@ -322,7 +322,7 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb } } if (args[4]) { - strategy = _PyLong_AsInt(args[4]); + strategy = PyLong_AsInt(args[4]); if (strategy == -1 && PyErr_Occurred()) { goto exit; } @@ -409,7 +409,7 @@ zlib_decompressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto skip_optional_pos; } if (args[0]) { - wbits = _PyLong_AsInt(args[0]); + wbits = PyLong_AsInt(args[0]); if (wbits == -1 && PyErr_Occurred()) { goto exit; } @@ -628,7 +628,7 @@ zlib_Compress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args, if (nargs < 1) { goto skip_optional_posonly; } - mode = _PyLong_AsInt(args[0]); + mode = PyLong_AsInt(args[0]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -1129,4 +1129,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=57ff7b511ab23132 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3eccb3f7265d53ba input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 33caca28224565..ecb08a86c2aab5 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -1203,7 +1203,7 @@ bytearray_hex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } } - bytes_per_sep = _PyLong_AsInt(args[1]); + bytes_per_sep = PyLong_AsInt(args[1]); if (bytes_per_sep == -1 && PyErr_Occurred()) { goto exit; } @@ -1256,7 +1256,7 @@ bytearray_reduce_ex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t n if (nargs < 1) { goto skip_optional; } - proto = _PyLong_AsInt(args[0]); + proto = PyLong_AsInt(args[0]); if (proto == -1 && PyErr_Occurred()) { goto exit; } @@ -1284,4 +1284,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=0817195f176cd8e3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cb94d7ac45f0a4b7 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 1ec0c95485a209..904ff07e97cde3 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -958,7 +958,7 @@ bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } } - bytes_per_sep = _PyLong_AsInt(args[1]); + bytes_per_sep = PyLong_AsInt(args[1]); if (bytes_per_sep == -1 && PyErr_Occurred()) { goto exit; } @@ -1060,4 +1060,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=bc4801bf1fa628f4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f4e85363c79c64d3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 511a8e4aaffea4..65801bd35156a6 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -57,27 +57,27 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!_PyArg_CheckPositional("code", PyTuple_GET_SIZE(args), 16, 18)) { goto exit; } - argcount = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); + argcount = PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); if (argcount == -1 && PyErr_Occurred()) { goto exit; } - posonlyargcount = _PyLong_AsInt(PyTuple_GET_ITEM(args, 1)); + posonlyargcount = PyLong_AsInt(PyTuple_GET_ITEM(args, 1)); if (posonlyargcount == -1 && PyErr_Occurred()) { goto exit; } - kwonlyargcount = _PyLong_AsInt(PyTuple_GET_ITEM(args, 2)); + kwonlyargcount = PyLong_AsInt(PyTuple_GET_ITEM(args, 2)); if (kwonlyargcount == -1 && PyErr_Occurred()) { goto exit; } - nlocals = _PyLong_AsInt(PyTuple_GET_ITEM(args, 3)); + nlocals = PyLong_AsInt(PyTuple_GET_ITEM(args, 3)); if (nlocals == -1 && PyErr_Occurred()) { goto exit; } - stacksize = _PyLong_AsInt(PyTuple_GET_ITEM(args, 4)); + stacksize = PyLong_AsInt(PyTuple_GET_ITEM(args, 4)); if (stacksize == -1 && PyErr_Occurred()) { goto exit; } - flags = _PyLong_AsInt(PyTuple_GET_ITEM(args, 5)); + flags = PyLong_AsInt(PyTuple_GET_ITEM(args, 5)); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -116,7 +116,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } qualname = PyTuple_GET_ITEM(args, 12); - firstlineno = _PyLong_AsInt(PyTuple_GET_ITEM(args, 13)); + firstlineno = PyLong_AsInt(PyTuple_GET_ITEM(args, 13)); if (firstlineno == -1 && PyErr_Occurred()) { goto exit; } @@ -231,7 +231,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje goto skip_optional_kwonly; } if (args[0]) { - co_argcount = _PyLong_AsInt(args[0]); + co_argcount = PyLong_AsInt(args[0]); if (co_argcount == -1 && PyErr_Occurred()) { goto exit; } @@ -240,7 +240,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[1]) { - co_posonlyargcount = _PyLong_AsInt(args[1]); + co_posonlyargcount = PyLong_AsInt(args[1]); if (co_posonlyargcount == -1 && PyErr_Occurred()) { goto exit; } @@ -249,7 +249,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[2]) { - co_kwonlyargcount = _PyLong_AsInt(args[2]); + co_kwonlyargcount = PyLong_AsInt(args[2]); if (co_kwonlyargcount == -1 && PyErr_Occurred()) { goto exit; } @@ -258,7 +258,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[3]) { - co_nlocals = _PyLong_AsInt(args[3]); + co_nlocals = PyLong_AsInt(args[3]); if (co_nlocals == -1 && PyErr_Occurred()) { goto exit; } @@ -267,7 +267,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[4]) { - co_stacksize = _PyLong_AsInt(args[4]); + co_stacksize = PyLong_AsInt(args[4]); if (co_stacksize == -1 && PyErr_Occurred()) { goto exit; } @@ -276,7 +276,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[5]) { - co_flags = _PyLong_AsInt(args[5]); + co_flags = PyLong_AsInt(args[5]); if (co_flags == -1 && PyErr_Occurred()) { goto exit; } @@ -285,7 +285,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[6]) { - co_firstlineno = _PyLong_AsInt(args[6]); + co_firstlineno = PyLong_AsInt(args[6]); if (co_firstlineno == -1 && PyErr_Occurred()) { goto exit; } @@ -455,7 +455,7 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n if (!args) { goto exit; } - oparg = _PyLong_AsInt(args[0]); + oparg = PyLong_AsInt(args[0]); if (oparg == -1 && PyErr_Occurred()) { goto exit; } @@ -464,4 +464,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=16c95266bbc4bc03 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0446968a1fbd13b2 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 0e6d9a4cad9e8f..a6561e2e54df80 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -112,7 +112,7 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar goto exit; } object = args[0]; - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -403,7 +403,7 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs goto skip_optional_pos; } } - bytes_per_sep = _PyLong_AsInt(args[1]); + bytes_per_sep = PyLong_AsInt(args[1]); if (bytes_per_sep == -1 && PyErr_Occurred()) { goto exit; } @@ -413,4 +413,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=a4f6992947bcaf25 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=28a632db32b44b63 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typeobject.c.h b/Objects/clinic/typeobject.c.h index 6feb5b1920fe20..1857eab17a793b 100644 --- a/Objects/clinic/typeobject.c.h +++ b/Objects/clinic/typeobject.c.h @@ -190,7 +190,7 @@ object___reduce_ex__(PyObject *self, PyObject *arg) PyObject *return_value = NULL; int protocol; - protocol = _PyLong_AsInt(arg); + protocol = PyLong_AsInt(arg); if (protocol == -1 && PyErr_Occurred()) { goto exit; } @@ -266,4 +266,4 @@ object___dir__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return object___dir___impl(self); } -/*[clinic end generated code: output=43533e6981550e9e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=baf5ec2093815a3f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index b9127e428f4c99..7edb4bc76099be 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -289,7 +289,7 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb if (!noptargs) { goto skip_optional_pos; } - tabsize = _PyLong_AsInt(args[0]); + tabsize = PyLong_AsInt(args[0]); if (tabsize == -1 && PyErr_Occurred()) { goto exit; } @@ -1504,4 +1504,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=ee76a1b49cd4cbb3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e1e1b136ef1b681 input=a9049054013a1b77]*/ diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index 49388cf043ced2..1e0df0958d6a4d 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -62,7 +62,7 @@ stringlib_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py if (!noptargs) { goto skip_optional_pos; } - tabsize = _PyLong_AsInt(args[0]); + tabsize = PyLong_AsInt(args[0]); if (tabsize == -1 && PyErr_Occurred()) { goto exit; } @@ -278,4 +278,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=d44a269805f6739e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=00e34c03331699fe input=a9049054013a1b77]*/ diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index 94e17306f2258e..af6710af9657e4 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -59,11 +59,11 @@ msvcrt_locking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("locking", nargs, 3, 3)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -105,11 +105,11 @@ msvcrt_setmode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("setmode", nargs, 2, 2)) { goto exit; } - fd = _PyLong_AsInt(args[0]); + fd = PyLong_AsInt(args[0]); if (fd == -1 && PyErr_Occurred()) { goto exit; } - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -154,7 +154,7 @@ msvcrt_open_osfhandle(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!handle && PyErr_Occurred()) { goto exit; } - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -189,7 +189,7 @@ msvcrt_get_osfhandle(PyObject *module, PyObject *arg) int fd; void *_return_value; - fd = _PyLong_AsInt(arg); + fd = PyLong_AsInt(arg); if (fd == -1 && PyErr_Occurred()) { goto exit; } @@ -514,7 +514,7 @@ msvcrt_CrtSetReportFile(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!_PyArg_CheckPositional("CrtSetReportFile", nargs, 2, 2)) { goto exit; } - type = _PyLong_AsInt(args[0]); + type = PyLong_AsInt(args[0]); if (type == -1 && PyErr_Occurred()) { goto exit; } @@ -561,11 +561,11 @@ msvcrt_CrtSetReportMode(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!_PyArg_CheckPositional("CrtSetReportMode", nargs, 2, 2)) { goto exit; } - type = _PyLong_AsInt(args[0]); + type = PyLong_AsInt(args[0]); if (type == -1 && PyErr_Occurred()) { goto exit; } - mode = _PyLong_AsInt(args[1]); + mode = PyLong_AsInt(args[1]); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -604,7 +604,7 @@ msvcrt_set_error_mode(PyObject *module, PyObject *arg) int mode; long _return_value; - mode = _PyLong_AsInt(arg); + mode = PyLong_AsInt(arg); if (mode == -1 && PyErr_Occurred()) { goto exit; } @@ -701,4 +701,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=9dd12bf210e362a4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f70de1b6d0e700cd input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 2aa0698185960b..a7401b9adc6880 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -401,7 +401,7 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto skip_optional_pos; } if (args[2]) { - reserved = _PyLong_AsInt(args[2]); + reserved = PyLong_AsInt(args[2]); if (reserved == -1 && PyErr_Occurred()) { goto exit; } @@ -409,7 +409,7 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto skip_optional_pos; } } - access = _PyLong_AsInt(args[3]); + access = PyLong_AsInt(args[3]); if (access == -1 && PyErr_Occurred()) { goto exit; } @@ -579,7 +579,7 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto skip_optional_pos; } if (args[2]) { - access = _PyLong_AsInt(args[2]); + access = PyLong_AsInt(args[2]); if (access == -1 && PyErr_Occurred()) { goto exit; } @@ -587,7 +587,7 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto skip_optional_pos; } } - reserved = _PyLong_AsInt(args[3]); + reserved = PyLong_AsInt(args[3]); if (reserved == -1 && PyErr_Occurred()) { goto exit; } @@ -695,7 +695,7 @@ winreg_EnumKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!clinic_HKEY_converter(_PyModule_GetState(module), args[0], &key)) { goto exit; } - index = _PyLong_AsInt(args[1]); + index = PyLong_AsInt(args[1]); if (index == -1 && PyErr_Occurred()) { goto exit; } @@ -752,7 +752,7 @@ winreg_EnumValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!clinic_HKEY_converter(_PyModule_GetState(module), args[0], &key)) { goto exit; } - index = _PyLong_AsInt(args[1]); + index = PyLong_AsInt(args[1]); if (index == -1 && PyErr_Occurred()) { goto exit; } @@ -1016,7 +1016,7 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje goto skip_optional_pos; } if (args[2]) { - reserved = _PyLong_AsInt(args[2]); + reserved = PyLong_AsInt(args[2]); if (reserved == -1 && PyErr_Occurred()) { goto exit; } @@ -1024,7 +1024,7 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje goto skip_optional_pos; } } - access = _PyLong_AsInt(args[3]); + access = PyLong_AsInt(args[3]); if (access == -1 && PyErr_Occurred()) { goto exit; } @@ -1133,7 +1133,7 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } if (args[2]) { - reserved = _PyLong_AsInt(args[2]); + reserved = PyLong_AsInt(args[2]); if (reserved == -1 && PyErr_Occurred()) { goto exit; } @@ -1141,7 +1141,7 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb goto skip_optional_pos; } } - access = _PyLong_AsInt(args[3]); + access = PyLong_AsInt(args[3]); if (access == -1 && PyErr_Occurred()) { goto exit; } @@ -1788,4 +1788,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=d2bf1f58ad07e5f8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4d0ec3e43e1b28f4 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winsound.c.h b/PC/clinic/winsound.c.h index 241d547c267a4c..d35261253557da 100644 --- a/PC/clinic/winsound.c.h +++ b/PC/clinic/winsound.c.h @@ -63,7 +63,7 @@ winsound_PlaySound(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto exit; } sound = args[0]; - flags = _PyLong_AsInt(args[1]); + flags = PyLong_AsInt(args[1]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -128,11 +128,11 @@ winsound_Beep(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec if (!args) { goto exit; } - frequency = _PyLong_AsInt(args[0]); + frequency = PyLong_AsInt(args[0]); if (frequency == -1 && PyErr_Occurred()) { goto exit; } - duration = _PyLong_AsInt(args[1]); + duration = PyLong_AsInt(args[1]); if (duration == -1 && PyErr_Occurred()) { goto exit; } @@ -196,7 +196,7 @@ winsound_MessageBeep(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!noptargs) { goto skip_optional_pos; } - type = _PyLong_AsInt(args[0]); + type = PyLong_AsInt(args[0]); if (type == -1 && PyErr_Occurred()) { goto exit; } @@ -206,4 +206,4 @@ winsound_MessageBeep(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=f70b7730127208d8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e63a6516d7a55cb8 input=a9049054013a1b77]*/ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index c71663919852ff..b39e412a7d6de9 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -194,7 +194,7 @@ warnings_warn_explicit(PyObject *module, PyObject *const *args, Py_ssize_t nargs goto exit; } filename = args[2]; - lineno = _PyLong_AsInt(args[3]); + lineno = PyLong_AsInt(args[3]); if (lineno == -1 && PyErr_Occurred()) { goto exit; } @@ -243,4 +243,4 @@ warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) { return warnings_filters_mutated_impl(module); } -/*[clinic end generated code: output=f8d67e0f75771c36 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0e2b367a662bf51b input=a9049054013a1b77]*/ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 7540de68280430..b0d6929aa50dc8 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -99,7 +99,7 @@ builtin___import__(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto skip_optional_pos; } } - level = _PyLong_AsInt(args[4]); + level = PyLong_AsInt(args[4]); if (level == -1 && PyErr_Occurred()) { goto exit; } @@ -242,7 +242,7 @@ builtin_chr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int i; - i = _PyLong_AsInt(arg); + i = PyLong_AsInt(arg); if (i == -1 && PyErr_Occurred()) { goto exit; } @@ -342,7 +342,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto skip_optional_pos; } if (args[3]) { - flags = _PyLong_AsInt(args[3]); + flags = PyLong_AsInt(args[3]); if (flags == -1 && PyErr_Occurred()) { goto exit; } @@ -360,7 +360,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[5]) { - optimize = _PyLong_AsInt(args[5]); + optimize = PyLong_AsInt(args[5]); if (optimize == -1 && PyErr_Occurred()) { goto exit; } @@ -372,7 +372,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_kwonly; } - feature_version = _PyLong_AsInt(args[6]); + feature_version = PyLong_AsInt(args[6]); if (feature_version == -1 && PyErr_Occurred()) { goto exit; } @@ -1212,4 +1212,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=daeee81b018824f4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bb2da8ccae4190e9 input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index dec9ea90b863c7..fd54033b7e7ac8 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -411,7 +411,7 @@ _imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int override; - override = _PyLong_AsInt(arg); + override = PyLong_AsInt(arg); if (override == -1 && PyErr_Occurred()) { goto exit; } @@ -442,7 +442,7 @@ _imp__override_multi_interp_extensions_check(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int override; - override = _PyLong_AsInt(arg); + override = PyLong_AsInt(arg); if (override == -1 && PyErr_Occurred()) { goto exit; } @@ -627,4 +627,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=a95ec234672280a2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d97b56ef622cb28a input=a9049054013a1b77]*/ diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index cf3984ca24bbfe..c97fb977ad3457 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -29,7 +29,7 @@ monitoring_use_tool_id(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (!_PyArg_CheckPositional("use_tool_id", nargs, 2, 2)) { goto exit; } - tool_id = _PyLong_AsInt(args[0]); + tool_id = PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } @@ -57,7 +57,7 @@ monitoring_free_tool_id(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int tool_id; - tool_id = _PyLong_AsInt(arg); + tool_id = PyLong_AsInt(arg); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } @@ -84,7 +84,7 @@ monitoring_get_tool(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int tool_id; - tool_id = _PyLong_AsInt(arg); + tool_id = PyLong_AsInt(arg); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } @@ -117,11 +117,11 @@ monitoring_register_callback(PyObject *module, PyObject *const *args, Py_ssize_t if (!_PyArg_CheckPositional("register_callback", nargs, 3, 3)) { goto exit; } - tool_id = _PyLong_AsInt(args[0]); + tool_id = PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - event = _PyLong_AsInt(args[1]); + event = PyLong_AsInt(args[1]); if (event == -1 && PyErr_Occurred()) { goto exit; } @@ -150,7 +150,7 @@ monitoring_get_events(PyObject *module, PyObject *arg) int tool_id; int _return_value; - tool_id = _PyLong_AsInt(arg); + tool_id = PyLong_AsInt(arg); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } @@ -185,11 +185,11 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("set_events", nargs, 2, 2)) { goto exit; } - tool_id = _PyLong_AsInt(args[0]); + tool_id = PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - event_set = _PyLong_AsInt(args[1]); + event_set = PyLong_AsInt(args[1]); if (event_set == -1 && PyErr_Occurred()) { goto exit; } @@ -222,7 +222,7 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { goto exit; } - tool_id = _PyLong_AsInt(args[0]); + tool_id = PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } @@ -260,12 +260,12 @@ monitoring_set_local_events(PyObject *module, PyObject *const *args, Py_ssize_t if (!_PyArg_CheckPositional("set_local_events", nargs, 3, 3)) { goto exit; } - tool_id = _PyLong_AsInt(args[0]); + tool_id = PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } code = args[1]; - event_set = _PyLong_AsInt(args[2]); + event_set = PyLong_AsInt(args[2]); if (event_set == -1 && PyErr_Occurred()) { goto exit; } @@ -308,4 +308,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=11cc0803875b3ffa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8baabc8341df3f0e input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index a593b980544b72..102e637ec18330 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -48,7 +48,7 @@ marshal_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - version = _PyLong_AsInt(args[2]); + version = PyLong_AsInt(args[2]); if (version == -1 && PyErr_Occurred()) { goto exit; } @@ -112,7 +112,7 @@ marshal_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - version = _PyLong_AsInt(args[1]); + version = PyLong_AsInt(args[1]); if (version == -1 && PyErr_Occurred()) { goto exit; } @@ -161,4 +161,4 @@ marshal_loads(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=12082d61d2942473 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f4ecfd941293f67 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a95a32f65edb89..1f751e48d3e0fe 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -444,7 +444,7 @@ sys_setrecursionlimit(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int new_limit; - new_limit = _PyLong_AsInt(arg); + new_limit = PyLong_AsInt(arg); if (new_limit == -1 && PyErr_Occurred()) { goto exit; } @@ -507,7 +507,7 @@ sys_set_coroutine_origin_tracking_depth(PyObject *module, PyObject *const *args, if (!args) { goto exit; } - depth = _PyLong_AsInt(args[0]); + depth = PyLong_AsInt(args[0]); if (depth == -1 && PyErr_Occurred()) { goto exit; } @@ -675,7 +675,7 @@ sys_setdlopenflags(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int new_val; - new_val = _PyLong_AsInt(arg); + new_val = PyLong_AsInt(arg); if (new_val == -1 && PyErr_Occurred()) { goto exit; } @@ -730,7 +730,7 @@ sys_mdebug(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flag; - flag = _PyLong_AsInt(arg); + flag = PyLong_AsInt(arg); if (flag == -1 && PyErr_Occurred()) { goto exit; } @@ -808,7 +808,7 @@ sys_set_int_max_str_digits(PyObject *module, PyObject *const *args, Py_ssize_t n if (!args) { goto exit; } - maxdigits = _PyLong_AsInt(args[0]); + maxdigits = PyLong_AsInt(args[0]); if (maxdigits == -1 && PyErr_Occurred()) { goto exit; } @@ -969,7 +969,7 @@ sys__getframe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - depth = _PyLong_AsInt(args[0]); + depth = PyLong_AsInt(args[0]); if (depth == -1 && PyErr_Occurred()) { goto exit; } @@ -1358,7 +1358,7 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_pos; } - depth = _PyLong_AsInt(args[0]); + depth = PyLong_AsInt(args[0]); if (depth == -1 && PyErr_Occurred()) { goto exit; } @@ -1412,4 +1412,4 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=41937e0843c68009 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6de02cd7d925d1de input=a9049054013a1b77]*/ diff --git a/Python/clinic/traceback.c.h b/Python/clinic/traceback.c.h index 3c344934971643..1a0eaf9e91b3c1 100644 --- a/Python/clinic/traceback.c.h +++ b/Python/clinic/traceback.c.h @@ -65,11 +65,11 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } tb_frame = (PyFrameObject *)fastargs[1]; - tb_lasti = _PyLong_AsInt(fastargs[2]); + tb_lasti = PyLong_AsInt(fastargs[2]); if (tb_lasti == -1 && PyErr_Occurred()) { goto exit; } - tb_lineno = _PyLong_AsInt(fastargs[3]); + tb_lineno = PyLong_AsInt(fastargs[3]); if (tb_lineno == -1 && PyErr_Occurred()) { goto exit; } @@ -78,4 +78,4 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=7bc9927e362fdfb7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6303f910c04227a4 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2c6fc4022e3809..a541c9f7e0760c 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3398,7 +3398,7 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'i': return """ - {paramname} = _PyLong_AsInt({argname}); + {paramname} = PyLong_AsInt({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} @@ -3586,7 +3586,7 @@ def converter_init( def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'i': return """ - {paramname} = _PyLong_AsInt({argname}); + {paramname} = PyLong_AsInt({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} From b32d4cad15f537f25063bcd56377e16df59d98db Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Aug 2023 01:01:30 +0200 Subject: [PATCH 319/598] gh-108444: Replace _PyLong_AsInt() with PyLong_AsInt() (#108459) Change generated by the command: sed -i -e 's!_PyLong_AsInt!PyLong_AsInt!g' \ $(find -name "*.c" -o -name "*.h") --- Modules/_csv.c | 2 +- Modules/_ctypes/stgdict.c | 2 +- Modules/_datetimemodule.c | 6 +++--- Modules/_io/fileio.c | 4 ++-- Modules/_io/winconsoleio.c | 2 +- Modules/_sqlite/connection.c | 2 +- Modules/faulthandler.c | 2 +- Modules/posixmodule.c | 6 +++--- Modules/readline.c | 2 +- Modules/socketmodule.c | 8 ++++---- Objects/bytesobject.c | 2 +- Objects/fileobject.c | 4 ++-- Objects/unicodeobject.c | 2 +- Python/assemble.c | 6 +++--- Python/ceval.c | 2 +- Python/flowgraph.c | 4 ++-- Python/initconfig.c | 2 +- Python/legacy_tracing.c | 6 +++--- 18 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Modules/_csv.c b/Modules/_csv.c index 24a57e362521db..9568334b8069c8 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -233,7 +233,7 @@ _set_int(const char *name, int *target, PyObject *src, int dflt) "\"%s\" must be an integer", name); return -1; } - value = _PyLong_AsInt(src); + value = PyLong_AsInt(src); if (value == -1 && PyErr_Occurred()) { return -1; } diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 54291a71667e38..9b0ca73a8b1751 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -401,7 +401,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } if (tmp) { - pack = _PyLong_AsInt(tmp); + pack = PyLong_AsInt(tmp); Py_DECREF(tmp); if (pack < 0) { if (!PyErr_Occurred() || diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9002a1de7fb5b7..454be6efc09d35 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1923,7 +1923,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } num = PyTuple_GET_ITEM(tuple, 1); /* us */ - us = _PyLong_AsInt(num); + us = PyLong_AsInt(num); num = NULL; if (us == -1 && PyErr_Occurred()) { goto Done; @@ -1941,7 +1941,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Py_DECREF(num); num = PyTuple_GET_ITEM(tuple, 1); /* seconds */ - s = _PyLong_AsInt(num); + s = PyLong_AsInt(num); num = NULL; if (s == -1 && PyErr_Occurred()) { goto Done; @@ -1951,7 +1951,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } num = Py_NewRef(PyTuple_GET_ITEM(tuple, 0)); /* leftover days */ - d = _PyLong_AsInt(num); + d = PyLong_AsInt(num); if (d == -1 && PyErr_Occurred()) { goto Done; } diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7fe37eee787e50..d52bcd50bd28c0 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -264,7 +264,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, self->fd = -1; } - fd = _PyLong_AsInt(nameobj); + fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, @@ -412,7 +412,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, goto error; } - self->fd = _PyLong_AsInt(fdobj); + self->fd = PyLong_AsInt(fdobj); Py_DECREF(fdobj); if (self->fd < 0) { if (!PyErr_Occurred()) { diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index a1ed7eb61e47b5..875c90c2ccd7bb 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -280,7 +280,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = -1; } - fd = _PyLong_AsInt(nameobj); + fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index e133977b28c378..a7780dedb55794 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1396,7 +1396,7 @@ authorizer_callback(void *ctx, int action, const char *arg1, } else { if (PyLong_Check(ret)) { - rc = _PyLong_AsInt(ret); + rc = PyLong_AsInt(ret); if (rc == -1 && PyErr_Occurred()) { print_or_clear_traceback(ctx); rc = SQLITE_DENY; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 5ec34d4e99c80d..aecb64dbb44647 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -116,7 +116,7 @@ faulthandler_get_fileno(PyObject **file_ptr) } } else if (PyLong_Check(file)) { - fd = _PyLong_AsInt(file); + fd = PyLong_AsInt(file); if (fd == -1 && PyErr_Occurred()) return -1; if (fd < 0) { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8026080912a7d8..cffa401d2ceb02 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6821,7 +6821,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg goto fail; } if (py_schedpolicy != Py_None) { - int schedpolicy = _PyLong_AsInt(py_schedpolicy); + int schedpolicy = PyLong_AsInt(py_schedpolicy); if (schedpolicy == -1 && PyErr_Occurred()) { goto fail; @@ -12484,7 +12484,7 @@ conv_confname(PyObject *arg, int *valuep, struct constdef *table, size_t tablesize) { if (PyLong_Check(arg)) { - int value = _PyLong_AsInt(arg); + int value = PyLong_AsInt(arg); if (value == -1 && PyErr_Occurred()) return 0; *valuep = value; @@ -15668,7 +15668,7 @@ os_waitstatus_to_exitcode_impl(PyObject *module, PyObject *status_obj) /*[clinic end generated code: output=db50b1b0ba3c7153 input=7fe2d7fdaea3db42]*/ { #ifndef MS_WINDOWS - int status = _PyLong_AsInt(status_obj); + int status = PyLong_AsInt(status_obj); if (status == -1 && PyErr_Occurred()) { return NULL; } diff --git a/Modules/readline.c b/Modules/readline.c index 6729a09cb0da5e..2531b236b6ef17 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1001,7 +1001,7 @@ on_hook(PyObject *func) if (r == Py_None) result = 0; else { - result = _PyLong_AsInt(r); + result = PyLong_AsInt(r); if (result == -1 && PyErr_Occurred()) goto error; } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index d1ac1ffec844a0..b4094b8c49a37f 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4879,17 +4879,17 @@ sock_sendmsg_afalg(PySocketSockObject *self, PyObject *args, PyObject *kwds) /* op is a required, keyword-only argument >= 0 */ if (opobj != NULL) { - op = _PyLong_AsInt(opobj); + op = PyLong_AsInt(opobj); } if (op < 0) { - /* override exception from _PyLong_AsInt() */ + /* override exception from PyLong_AsInt() */ PyErr_SetString(PyExc_TypeError, "Invalid or missing argument 'op'"); goto finally; } /* assoclen is optional but must be >= 0 */ if (assoclenobj != NULL) { - assoclen = _PyLong_AsInt(assoclenobj); + assoclen = PyLong_AsInt(assoclenobj); if (assoclen == -1 && PyErr_Occurred()) { goto finally; } @@ -5007,7 +5007,7 @@ sock_shutdown(PySocketSockObject *s, PyObject *arg) int how; int res; - how = _PyLong_AsInt(arg); + how = PyLong_AsInt(arg); if (how == -1 && PyErr_Occurred()) return NULL; Py_BEGIN_ALLOW_THREADS diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a2c1f4cf359f91..c3a31bec822c37 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -753,7 +753,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, "* wants int"); goto error; } - prec = _PyLong_AsInt(v); + prec = PyLong_AsInt(v); if (prec == -1 && PyErr_Occurred()) goto error; if (prec < 0) diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 751fb69d0891cf..9c240250218838 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -174,7 +174,7 @@ PyObject_AsFileDescriptor(PyObject *o) PyObject *meth; if (PyLong_Check(o)) { - fd = _PyLong_AsInt(o); + fd = PyLong_AsInt(o); } else if (PyObject_GetOptionalAttr(o, &_Py_ID(fileno), &meth) < 0) { return -1; @@ -186,7 +186,7 @@ PyObject_AsFileDescriptor(PyObject *o) return -1; if (PyLong_Check(fno)) { - fd = _PyLong_AsInt(fno); + fd = PyLong_AsInt(fno); Py_DECREF(fno); } else { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6bbc2af258e675..2f6f41367d3888 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14034,7 +14034,7 @@ unicode_format_arg_parse(struct unicode_formatter_t *ctx, "* wants int"); return -1; } - arg->prec = _PyLong_AsInt(v); + arg->prec = PyLong_AsInt(v); if (arg->prec == -1 && PyErr_Occurred()) return -1; if (arg->prec < 0) diff --git a/Python/assemble.c b/Python/assemble.c index c770fd108d41d6..b6fb432aed4a3b 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -476,7 +476,7 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_varnames, &pos, &k, &v)) { - int offset = _PyLong_AsInt(v); + int offset = PyLong_AsInt(v); if (offset == -1 && PyErr_Occurred()) { return ERROR; } @@ -513,7 +513,7 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, continue; } - int offset = _PyLong_AsInt(v); + int offset = PyLong_AsInt(v); if (offset == -1 && PyErr_Occurred()) { return ERROR; } @@ -525,7 +525,7 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, pos = 0; while (PyDict_Next(umd->u_freevars, &pos, &k, &v)) { - int offset = _PyLong_AsInt(v); + int offset = PyLong_AsInt(v); if (offset == -1 && PyErr_Occurred()) { return ERROR; } diff --git a/Python/ceval.c b/Python/ceval.c index 55dfe6b1efc475..a56d31ea073639 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2416,7 +2416,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, /* Fast path for not overloaded __import__. */ if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { - int ilevel = _PyLong_AsInt(level); + int ilevel = PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index e620e7cf1b9e96..55b871dd627364 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2412,13 +2412,13 @@ build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) continue; } - int argoffset = _PyLong_AsInt(varindex); + int argoffset = PyLong_AsInt(varindex); Py_DECREF(varindex); if (argoffset == -1 && PyErr_Occurred()) { goto error; } - int oldindex = _PyLong_AsInt(cellindex); + int oldindex = PyLong_AsInt(cellindex); if (oldindex == -1 && PyErr_Occurred()) { goto error; } diff --git a/Python/initconfig.c b/Python/initconfig.c index 39d21adf546466..3281b3c6aea45f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1097,7 +1097,7 @@ config_dict_get_int(PyObject *dict, const char *name, int *result) if (item == NULL) { return -1; } - int value = _PyLong_AsInt(item); + int value = PyLong_AsInt(item); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 7774d10b10172b..17a13b1361f992 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -256,7 +256,7 @@ sys_trace_line_func( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 2); - int line = _PyLong_AsInt(args[1]); + int line = PyLong_AsInt(args[1]); assert(line >= 0); PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { @@ -282,9 +282,9 @@ sys_trace_jump_func( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 3); - int from = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); + int from = PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); assert(from >= 0); - int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); + int to = PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); assert(to >= 0); if (to > from) { /* Forward jump */ From d6ac5c7b105fe57266bd71248e3ada41fedb5ba9 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 25 Aug 2023 08:03:26 +0900 Subject: [PATCH 320/598] gh-107265: Ensure _PyCode_Quicken does not handle ENTER_EXECUTOR (gh-108460) --- Python/specialize.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/specialize.c b/Python/specialize.c index a467f163f2ca9d..fac3c3ea57d004 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -300,6 +300,8 @@ _PyCode_Quicken(PyCodeObject *code) for (int i = 0; i < Py_SIZE(code); i++) { opcode = _Py_GetBaseOpcode(code, i); assert(opcode < MIN_INSTRUMENTED_OPCODE); + // _PyCode_Quicken is only called when initializing a fresh code object. + assert(opcode != ENTER_EXECUTOR); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); From 546cab84448b892c92e68d9c1a3d3b58c13b3463 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Aug 2023 01:35:47 +0200 Subject: [PATCH 321/598] gh-106320: Remove private _PyTraceback functions (#108453) Move private functions to the internal C API (pycore_traceback.h): * _Py_DisplaySourceLine() * _PyTraceback_Add() --- Include/cpython/traceback.h | 3 --- Include/internal/pycore_traceback.h | 6 ++++++ Modules/_ctypes/callproc.c | 6 ++++-- Modules/pyexpat.c | 1 + Python/_warnings.c | 5 ++--- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index a4e087b2b4eced..81c51944f136f2 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,6 +11,3 @@ struct _traceback { int tb_lasti; int tb_lineno; }; - -PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); -PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 21fb4a25a0face..01b3f5b6b26d2e 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +// Export for '_ctypes' shared extension +PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); + +// Export for 'pyexact' shared extension +PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); + /* Write the Python traceback into the file 'fd'. For example: Traceback (most recent call first): diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f9535db4f57c0e..fc08c42bd3574a 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -101,8 +101,10 @@ module _ctypes #define DONT_USE_SEH #endif -#include "pycore_runtime.h" // _PyRuntime -#include "pycore_global_objects.h" // _Py_ID() +#include "pycore_runtime.h" // _PyRuntime +#include "pycore_global_objects.h"// _Py_ID() +#include "pycore_traceback.h" // _PyTraceback_Add() + #include "clinic/callproc.c.h" #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index a8ce84c0bb9f05..52dd06cd3c8181 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_import.h" // _PyImport_SetModule() #include "pycore_pyhash.h" // _Py_HashSecret +#include "pycore_traceback.h" // _PyTraceback_Add() #include #include // offsetof() diff --git a/Python/_warnings.c b/Python/_warnings.c index 9e562d7ba7f6a7..4b7fb888247145 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1,13 +1,12 @@ #include "Python.h" #include "pycore_dict.h" // _PyDict_GetItemWithError() -#include "pycore_frame.h" -#include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.warnings #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_pyerrors.h" +#include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_traceback.h" // _Py_DisplaySourceLine() #include "clinic/_warnings.c.h" From ddf66b54edea1ea59fdf8a496ed0b64e16424375 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 24 Aug 2023 17:36:00 -0700 Subject: [PATCH 322/598] gh-106581: Split CALL_BOUND_METHOD_EXACT_ARGS into uops (#108462) Instead of using `GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS)` we just add the macro elements of the latter to the macro for the former. This requires lengthening the uops array in struct opcode_macro_expansion. (It also required changes to stacking.py that were merged already.) --- Include/internal/pycore_opcode_metadata.h | 41 ++++++---- Python/abstract_interp_cases.c.h | 10 +++ Python/bytecodes.c | 29 ++++--- Python/executor_cases.c.h | 26 ++++++ Python/generated_cases.c.h | 98 ++++++++++++++++++++--- Tools/cases_generator/generate_cases.py | 2 +- 6 files changed, 171 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index cc8894ad53933f..bf7e9dcbffad23 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -57,16 +57,18 @@ #define _ITER_CHECK_RANGE 329 #define _IS_ITER_EXHAUSTED_RANGE 330 #define _ITER_NEXT_RANGE 331 -#define _CHECK_PEP_523 332 -#define _CHECK_FUNCTION_EXACT_ARGS 333 -#define _CHECK_STACK_SPACE 334 -#define _INIT_CALL_PY_EXACT_ARGS 335 -#define _PUSH_FRAME 336 -#define _POP_JUMP_IF_FALSE 337 -#define _POP_JUMP_IF_TRUE 338 -#define JUMP_TO_TOP 339 -#define SAVE_CURRENT_IP 340 -#define INSERT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 332 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 333 +#define _CHECK_PEP_523 334 +#define _CHECK_FUNCTION_EXACT_ARGS 335 +#define _CHECK_STACK_SPACE 336 +#define _INIT_CALL_PY_EXACT_ARGS 337 +#define _PUSH_FRAME 338 +#define _POP_JUMP_IF_FALSE 339 +#define _POP_JUMP_IF_TRUE 340 +#define JUMP_TO_TOP 341 +#define SAVE_CURRENT_IP 342 +#define INSERT 343 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -484,7 +486,9 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case CALL: return oparg + 2; - case CALL_BOUND_METHOD_EXACT_ARGS: + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: return oparg + 2; case _CHECK_PEP_523: return 0; @@ -496,6 +500,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg + 2; case _PUSH_FRAME: return 1; + case CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; case CALL_PY_EXACT_ARGS: return oparg + 2; case CALL_PY_WITH_DEFAULTS: @@ -1012,8 +1018,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case CALL: return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return 1; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; case _CHECK_PEP_523: return 0; case _CHECK_FUNCTION_EXACT_ARGS: @@ -1024,6 +1032,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case _PUSH_FRAME: return 1; + case CALL_BOUND_METHOD_EXACT_ARGS: + return 1; case CALL_PY_EXACT_ARGS: return 1; case CALL_PY_WITH_DEFAULTS: @@ -1163,7 +1173,7 @@ struct opcode_metadata { struct opcode_macro_expansion { int nuops; - struct { int16_t uop; int8_t size; int8_t offset; } uops[8]; + struct { int16_t uop; int8_t size; int8_t offset; } uops[12]; }; #define OPARG_FULL 0 @@ -1518,6 +1528,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { SAVE_IP, 7, 3 }, { SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { SAVE_IP, 7, 3 }, { SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_NO_KW_TYPE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TYPE_1, 0, 0 } } }, [CALL_NO_KW_STR_1] = { .nuops = 1, .uops = { { CALL_NO_KW_STR_1, 0, 0 } } }, @@ -1577,6 +1588,8 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 1b99b929fa8014..07e8a711575c5b 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -621,6 +621,16 @@ break; } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - oparg)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); + break; + } + case _CHECK_PEP_523: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ae459cabaddc05..a5cb117c7631c6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2940,19 +2940,18 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // Start out with [NULL, bound_method, arg1, arg2, ...] - // Transform to [callable, self, arg1, arg2, ...] - // Then fall through to CALL_PY_EXACT_ARGS - inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, callable, null, unused[oparg] -- unused)) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // self_or_null - PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // callable + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization Py_DECREF(callable); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } op(_CHECK_PEP_523, (--)) { @@ -3010,6 +3009,18 @@ dummy_func( #endif } + macro(CALL_BOUND_METHOD_EXACT_ARGS) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _INIT_CALL_PY_EXACT_ARGS + + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _PUSH_FRAME; + macro(CALL_PY_EXACT_ARGS) = unused/1 + // Skip over the counter _CHECK_PEP_523 + diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 89a5bbfecded0e..85c60c276e0292 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2190,6 +2190,32 @@ break; } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + PyObject *null; + PyObject *callable; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + PyObject *callable; + PyObject *func; + PyObject *self; + callable = stack_pointer[-2 - oparg]; + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; + break; + } + case _CHECK_PEP_523: { DEOPT_IF(tstate->interp->eval_frame, CALL); break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1724d11231727f..4aa16f8311a2a0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3795,23 +3795,99 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *null; PyObject *callable; + PyObject *self; + PyObject *self_or_null; + PyObject *func; + PyObject **args; + _PyInterpreterFrame *new_frame; + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // self_or_null - PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // callable - Py_DECREF(callable); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + { + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&next_instr[1].cache); + ASSERT_KWNAMES_IS_NULL(); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = stack_pointer - oparg; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // SAVE_CURRENT_IP + next_instr += 3; + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _PUSH_FRAME STACK_SHRINK(oparg); - STACK_SHRINK(1); + STACK_SHRINK(2); + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + frame->return_offset = 0; + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + #if TIER_ONE + goto start_frame; + #endif + #if TIER_TWO + if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(CALL_PY_EXACT_ARGS) { - PREDICTED(CALL_PY_EXACT_ARGS); PyObject *self_or_null; PyObject *callable; PyObject **args; diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 8f9a6502e529c7..9400a0bce462c8 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -444,7 +444,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No with self.out.block("struct opcode_macro_expansion", ";"): self.out.emit("int nuops;") self.out.emit( - "struct { int16_t uop; int8_t size; int8_t offset; } uops[8];" + "struct { int16_t uop; int8_t size; int8_t offset; } uops[12];" ) self.out.emit("") From 135098743a0fae0efbcd98e35458e5bc721702e9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 25 Aug 2023 06:56:54 +0100 Subject: [PATCH 323/598] GH-108202: Document ``calendar`` exceptions (#108398) --- Doc/library/calendar.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 07d04a1c7b582a..3c097f4cf7abd9 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -469,6 +469,29 @@ The :mod:`calendar` module exports the following data attributes: Aliases for day numbers, where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. + +The :mod:`calendar` module defines the following exceptions: + +.. exception:: IllegalMonthError(month) + + A subclass of :exc:`ValueError`, + raised when the given month number is outside of the range 1-12 (inclusive). + + .. attribute:: month + + The invalid month number. + + +.. exception:: IllegalWeekdayError(weekday) + + A subclass of :exc:`ValueError`, + raised when the given weekday number is outside of the range 0-6 (inclusive). + + .. attribute:: weekday + + The invalid weekday number. + + .. seealso:: Module :mod:`datetime` From 8d4052075ec43ac1ded7d2fa55c474295410bbdc Mon Sep 17 00:00:00 2001 From: Peeyush Aggarwal Date: Fri, 25 Aug 2023 13:45:26 +0530 Subject: [PATCH 324/598] gh-103384: Generalize the regex pattern `BaseConfigurator.INDEX_PATTERN` to allow spaces and non-alphanumeric characters in keys. (GH-103391) Co-authored-by: Vinay Sajip Co-authored-by: Alex Waygood Co-authored-by: Erlend E. Aasland --- Doc/library/logging.config.rst | 3 +- Lib/logging/config.py | 2 +- Lib/test/test_logging.py | 48 ++++++++++++++++++- ...-04-09-05-30-41.gh-issue-103384.zAV7iB.rst | 1 + 4 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-09-05-30-41.gh-issue-103384.zAV7iB.rst diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 448978f43b6d13..53fbd073c26702 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -685,7 +685,8 @@ resolve to ``'dev_team@domain.tld'`` and the string ``'support_team@domain.tld'``. The ``subject`` value could be accessed using either ``'cfg://handlers.email.subject'`` or, equivalently, ``'cfg://handlers.email[subject]'``. The latter form only needs to be -used if the key contains spaces or non-alphanumeric characters. If an +used if the key contains spaces or non-alphanumeric characters. Please note +that the characters ``[`` and ``]`` are not allowed in the keys. If an index value consists only of decimal digits, access will be attempted using the corresponding integer value, falling back to the string value if needed. diff --git a/Lib/logging/config.py b/Lib/logging/config.py index a68281d3e359fd..41283f4d627267 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -378,7 +378,7 @@ class BaseConfigurator(object): WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') - INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') + INDEX_PATTERN = re.compile(r'^\[([^\[\]]*)\]\s*') DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index f26846f9663e5c..c2e8ff5d463607 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3662,7 +3662,28 @@ def test_baseconfig(self): d = { 'atuple': (1, 2, 3), 'alist': ['a', 'b', 'c'], - 'adict': {'d': 'e', 'f': 3 }, + 'adict': { + 'd': 'e', 'f': 3 , + 'alpha numeric 1 with spaces' : 5, + 'aplha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric ] 1 with spaces' : 15, + 'aplha ]] numeric 1 %( - © ©ß¯]' : 19, + ' aplha [ numeric 1 %( - © ©ß¯] ' : 11, + ' aplha ' : 32, + '' : 10, + 'nest4' : { + 'd': 'e', 'f': 3 , + 'alpha numeric 1 with spaces' : 5, + 'aplha numeric 1 %( - © ©ß¯' : 9, + '' : 10, + 'somelist' : ('g', ('h', 'i'), 'j'), + 'somedict' : { + 'a' : 1, + 'a with 1 and space' : 3, + 'a with ( and space' : 4, + } + } + }, 'nest1': ('g', ('h', 'i'), 'j'), 'nest2': ['k', ['l', 'm'], 'n'], 'nest3': ['o', 'cfg://alist', 'p'], @@ -3674,11 +3695,36 @@ def test_baseconfig(self): self.assertEqual(bc.convert('cfg://nest2[1][1]'), 'm') self.assertEqual(bc.convert('cfg://adict.d'), 'e') self.assertEqual(bc.convert('cfg://adict[f]'), 3) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 with spaces]'), 5) + self.assertEqual(bc.convert('cfg://adict[aplha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[]'), 10) + self.assertEqual(bc.convert('cfg://adict.nest4.d'), 'e') + self.assertEqual(bc.convert('cfg://adict.nest4[d]'), 'e') + self.assertEqual(bc.convert('cfg://adict[nest4].d'), 'e') + self.assertEqual(bc.convert('cfg://adict[nest4][f]'), 3) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 with spaces]'), 5) + self.assertEqual(bc.convert('cfg://adict[nest4][aplha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[nest4][]'), 10) + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][0]'), 'g') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][0]'), 'h') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][1]'), 'i') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][2]'), 'j') + self.assertEqual(bc.convert('cfg://adict[nest4].somedict.a'), 1) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a]'), 1) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a with 1 and space]'), 3) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a with ( and space]'), 4) + self.assertEqual(bc.convert('cfg://adict.nest4.somelist[1][1]'), 'i') + self.assertEqual(bc.convert('cfg://adict.nest4.somelist[2]'), 'j') + self.assertEqual(bc.convert('cfg://adict.nest4.somedict.a'), 1) + self.assertEqual(bc.convert('cfg://adict.nest4.somedict[a]'), 1) v = bc.convert('cfg://nest3') self.assertEqual(v.pop(1), ['a', 'b', 'c']) self.assertRaises(KeyError, bc.convert, 'cfg://nosuch') self.assertRaises(ValueError, bc.convert, 'cfg://!') self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') + self.assertRaises(KeyError, bc.convert, 'cfg://adict[alpha numeric ] 1 with spaces]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ aplha ]] numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ aplha [ numeric 1 %( - © ©ß¯] ]') def test_namedtuple(self): # see bpo-39142 diff --git a/Misc/NEWS.d/next/Library/2023-04-09-05-30-41.gh-issue-103384.zAV7iB.rst b/Misc/NEWS.d/next/Library/2023-04-09-05-30-41.gh-issue-103384.zAV7iB.rst new file mode 100644 index 00000000000000..3e9d3cf9ae92b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-09-05-30-41.gh-issue-103384.zAV7iB.rst @@ -0,0 +1 @@ +Generalize the regex pattern ``BaseConfigurator.INDEX_PATTERN`` to allow spaces and non-alphanumeric characters in keys. From 24aa249a6633249570978d6aae6f7b21581ee085 Mon Sep 17 00:00:00 2001 From: Corvin Date: Fri, 25 Aug 2023 04:31:26 -0400 Subject: [PATCH 325/598] gh-107932: Fix dis module for bytecode that does not have an associated source line (GH-107988) --- Doc/library/dis.rst | 15 +- Lib/dis.py | 53 +- Lib/test/test_dis.py | 803 +++++++++--------- Misc/ACKS | 1 + ...-08-16-21-20-55.gh-issue-107932.I7hFsp.rst | 1 + 5 files changed, 468 insertions(+), 405 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-16-21-20-55.gh-issue-107932.I7hFsp.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 0b44d160de58a7..7c3363b143ab58 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -297,6 +297,9 @@ operation is being performed, so the intermediate analysis object isn't useful: The :pep:`626` ``co_lines`` method is used instead of the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object. + .. versionchanged:: 3.13 + Line numbers can be ``None`` for bytecode that does not map to source lines. + .. function:: findlabels(code) @@ -402,7 +405,12 @@ details of bytecode instructions as :class:`Instruction` instances: .. data:: starts_line - line started by this opcode (if any), otherwise ``None`` + ``True`` if this opcode starts a source line, otherwise ``False`` + + + .. data:: line_number + + source line number associated with this opcode (if any), otherwise ``None`` .. data:: is_jump_target @@ -429,8 +437,11 @@ details of bytecode instructions as :class:`Instruction` instances: .. versionchanged:: 3.13 + Changed field ``starts_line``. + Added fields ``start_offset``, ``cache_offset``, ``end_offset``, - ``baseopname``, ``baseopcode``, ``jump_target`` and ``oparg``. + ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, and + ``line_number``. .. class:: Positions diff --git a/Lib/dis.py b/Lib/dis.py index 4f4e77f32503df..7e4792e8a8dc62 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -262,6 +262,7 @@ def show_code(co, *, file=None): 'offset', 'start_offset', 'starts_line', + 'line_number', 'is_jump_target', 'positions' ], @@ -278,7 +279,8 @@ def show_code(co, *, file=None): "Start index of operation within bytecode sequence, including extended args if present; " "otherwise equal to Instruction.offset" ) -_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" +_Instruction.starts_line.__doc__ = "True if this opcode starts a source line, otherwise False" +_Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" _Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" _Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" @@ -321,7 +323,8 @@ class Instruction(_Instruction): offset - start index of operation within bytecode sequence start_offset - start index of operation within bytecode sequence including extended args if present; otherwise equal to Instruction.offset - starts_line - line started by this opcode (if any), otherwise None + starts_line - True if this opcode starts a source line, otherwise False + line_number - source line number associated with this opcode (if any), otherwise None is_jump_target - True if other code jumps to here, otherwise False positions - Optional dis.Positions object holding the span of source code covered by this instruction @@ -376,9 +379,10 @@ def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4): fields = [] # Column: Source code line number if lineno_width: - if self.starts_line is not None: - lineno_fmt = "%%%dd" % lineno_width - fields.append(lineno_fmt % self.starts_line) + if self.starts_line: + lineno_fmt = "%%%dd" if self.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + fields.append(lineno_fmt % self.line_number) else: fields.append(' ' * lineno_width) # Column: Current instruction indicator @@ -527,12 +531,18 @@ def _get_instructions_bytes(code, varname_from_oparg=None, for start, end, target, _, _ in exception_entries: for i in range(start, end): labels.add(target) - starts_line = None + starts_line = False + local_line_number = None + line_number = None for offset, start_offset, op, arg in _unpack_opargs(original_code): if linestarts is not None: - starts_line = linestarts.get(offset, None) - if starts_line is not None: - starts_line += line_offset + starts_line = offset in linestarts + if starts_line: + local_line_number = linestarts[offset] + if local_line_number is not None: + line_number = local_line_number + line_offset + else: + line_number = None is_jump_target = offset in labels argval = None argrepr = '' @@ -599,7 +609,8 @@ def _get_instructions_bytes(code, varname_from_oparg=None, argrepr = _intrinsic_2_descs[arg] yield Instruction(_all_opname[op], op, arg, argval, argrepr, - offset, start_offset, starts_line, is_jump_target, positions) + offset, start_offset, starts_line, line_number, + is_jump_target, positions) if not caches: continue if not show_caches: @@ -618,7 +629,7 @@ def _get_instructions_bytes(code, varname_from_oparg=None, else: argrepr = "" yield Instruction( - "CACHE", CACHE, 0, None, argrepr, offset, offset, None, False, + "CACHE", CACHE, 0, None, argrepr, offset, offset, False, None, False, Positions(*next(co_positions, ())) ) @@ -651,13 +662,21 @@ def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None, *, file=None, line_offset=0, exception_entries=(), co_positions=None, show_caches=False, original_code=None): # Omit the line number column entirely if we have no line number info - show_lineno = bool(linestarts) + if bool(linestarts): + linestarts_ints = [line for line in linestarts.values() if line is not None] + show_lineno = len(linestarts_ints) > 0 + else: + show_lineno = False + if show_lineno: - maxlineno = max(linestarts.values()) + line_offset + maxlineno = max(linestarts_ints) + line_offset if maxlineno >= 1000: lineno_width = len(str(maxlineno)) else: lineno_width = 3 + + if lineno_width < len(str(None)) and None in linestarts.values(): + lineno_width = len(str(None)) else: lineno_width = 0 maxoffset = len(code) - 2 @@ -673,7 +692,7 @@ def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None, show_caches=show_caches, original_code=original_code): new_source_line = (show_lineno and - instr.starts_line is not None and + instr.starts_line and instr.offset > 0) if new_source_line: print(file=file) @@ -755,10 +774,12 @@ def findlinestarts(code): """Find the offsets in a byte code which are start of lines in the source. Generate pairs (offset, lineno) + lineno will be an integer or None the offset does not have a source line. """ - lastline = None + + lastline = False # None is a valid line number for start, end, line in code.co_lines(): - if line is not None and line != lastline: + if line is not lastline: lastline = line yield start, line return diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index aa967178c1dc39..f51c7dae8a4ce7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -388,43 +388,46 @@ def wrap_func_w_kwargs(): """ dis_traceback = """\ -%3d RESUME 0 +%4d RESUME 0 -%3d NOP +%4d NOP -%3d LOAD_CONST 1 (1) - LOAD_CONST 2 (0) - --> BINARY_OP 11 (/) - POP_TOP +%4d LOAD_CONST 1 (1) + LOAD_CONST 2 (0) + --> BINARY_OP 11 (/) + POP_TOP -%3d LOAD_FAST_CHECK 1 (tb) - RETURN_VALUE - >> PUSH_EXC_INFO - -%3d LOAD_GLOBAL 0 (Exception) - CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to 80) - STORE_FAST 0 (e) - -%3d LOAD_FAST 0 (e) - LOAD_ATTR 2 (__traceback__) - STORE_FAST 1 (tb) - POP_EXCEPT - LOAD_CONST 0 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) - -%3d LOAD_FAST 1 (tb) - RETURN_VALUE - >> LOAD_CONST 0 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) - RERAISE 1 - -%3d >> RERAISE 0 - >> COPY 3 - POP_EXCEPT - RERAISE 1 +%4d LOAD_FAST_CHECK 1 (tb) + RETURN_VALUE + +None >> PUSH_EXC_INFO + +%4d LOAD_GLOBAL 0 (Exception) + CHECK_EXC_MATCH + POP_JUMP_IF_FALSE 23 (to 80) + STORE_FAST 0 (e) + +%4d LOAD_FAST 0 (e) + LOAD_ATTR 2 (__traceback__) + STORE_FAST 1 (tb) + POP_EXCEPT + LOAD_CONST 0 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + +%4d LOAD_FAST 1 (tb) + RETURN_VALUE + +None >> LOAD_CONST 0 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + RERAISE 1 + +%4d >> RERAISE 0 + +None >> COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: 4 rows """ % (TRACEBACK_CODE.co_firstlineno, @@ -467,41 +470,42 @@ def _with(c): y = 2 dis_with = """\ -%3d RESUME 0 +%4d RESUME 0 -%3d LOAD_FAST 0 (c) - BEFORE_WITH - POP_TOP +%4d LOAD_FAST 0 (c) + BEFORE_WITH + POP_TOP -%3d LOAD_CONST 1 (1) - STORE_FAST 1 (x) +%4d LOAD_CONST 1 (1) + STORE_FAST 1 (x) -%3d LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - CALL 2 - POP_TOP +%4d LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 2 + POP_TOP -%3d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) -%3d >> PUSH_EXC_INFO - WITH_EXCEPT_START - TO_BOOL - POP_JUMP_IF_TRUE 1 (to 50) - RERAISE 2 - >> POP_TOP - POP_EXCEPT - POP_TOP - POP_TOP +%4d >> PUSH_EXC_INFO + WITH_EXCEPT_START + TO_BOOL + POP_JUMP_IF_TRUE 1 (to 50) + RERAISE 2 + >> POP_TOP + POP_EXCEPT + POP_TOP + POP_TOP -%3d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) - >> COPY 3 - POP_EXCEPT - RERAISE 1 +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + +None >> COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: 2 rows """ % (_with.__code__.co_firstlineno, @@ -519,79 +523,89 @@ async def _asyncwith(c): y = 2 dis_asyncwith = """\ -%3d RETURN_GENERATOR - POP_TOP - RESUME 0 - -%3d LOAD_FAST 0 (c) - BEFORE_ASYNC_WITH - GET_AWAITABLE 1 - LOAD_CONST 0 (None) - >> SEND 3 (to 24) - YIELD_VALUE 2 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) - >> END_SEND - POP_TOP +%4d RETURN_GENERATOR -%3d LOAD_CONST 1 (1) - STORE_FAST 1 (x) - -%3d LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - CALL 2 - GET_AWAITABLE 2 - LOAD_CONST 0 (None) - >> SEND 3 (to 60) - YIELD_VALUE 2 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 50) - >> END_SEND - POP_TOP +None POP_TOP -%3d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) +%4d RESUME 0 -%3d >> CLEANUP_THROW - JUMP_BACKWARD 26 (to 24) - >> CLEANUP_THROW - JUMP_BACKWARD 11 (to 60) - >> PUSH_EXC_INFO - WITH_EXCEPT_START - GET_AWAITABLE 2 - LOAD_CONST 0 (None) - >> SEND 4 (to 102) - YIELD_VALUE 3 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) - >> CLEANUP_THROW - >> END_SEND - TO_BOOL - POP_JUMP_IF_TRUE 1 (to 116) - RERAISE 2 - >> POP_TOP - POP_EXCEPT - POP_TOP - POP_TOP +%4d LOAD_FAST 0 (c) + BEFORE_ASYNC_WITH + GET_AWAITABLE 1 + LOAD_CONST 0 (None) + >> SEND 3 (to 24) + YIELD_VALUE 2 + RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) + >> END_SEND + POP_TOP -%3d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) - >> COPY 3 - POP_EXCEPT - RERAISE 1 - >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) - RERAISE 1 +%4d LOAD_CONST 1 (1) + STORE_FAST 1 (x) + +%4d LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 2 + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + >> SEND 3 (to 60) + YIELD_VALUE 2 + RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to 50) + >> END_SEND + POP_TOP + +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + +%4d >> CLEANUP_THROW + +None JUMP_BACKWARD 26 (to 24) + +%4d >> CLEANUP_THROW + +None JUMP_BACKWARD 11 (to 60) + +%4d >> PUSH_EXC_INFO + WITH_EXCEPT_START + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + >> SEND 4 (to 102) + YIELD_VALUE 3 + RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) + >> CLEANUP_THROW + >> END_SEND + TO_BOOL + POP_JUMP_IF_TRUE 1 (to 116) + RERAISE 2 + >> POP_TOP + POP_EXCEPT + POP_TOP + POP_TOP + +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + +None >> COPY 3 + POP_EXCEPT + RERAISE 1 + >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 ExceptionTable: 12 rows """ % (_asyncwith.__code__.co_firstlineno, + _asyncwith.__code__.co_firstlineno, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 2, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 3, _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 3, ) @@ -609,61 +623,67 @@ def _tryfinallyconst(b): b() dis_tryfinally = """\ -%3d RESUME 0 +%4d RESUME 0 -%3d NOP +%4d NOP -%3d LOAD_FAST 0 (a) +%4d LOAD_FAST 0 (a) -%3d LOAD_FAST 1 (b) - PUSH_NULL - CALL 0 - POP_TOP - RETURN_VALUE - >> PUSH_EXC_INFO - LOAD_FAST 1 (b) - PUSH_NULL - CALL 0 - POP_TOP - RERAISE 0 - >> COPY 3 - POP_EXCEPT - RERAISE 1 +%4d LOAD_FAST 1 (b) + PUSH_NULL + CALL 0 + POP_TOP + RETURN_VALUE + +None >> PUSH_EXC_INFO + +%4d LOAD_FAST 1 (b) + PUSH_NULL + CALL 0 + POP_TOP + RERAISE 0 + >> COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: 2 rows """ % (_tryfinally.__code__.co_firstlineno, _tryfinally.__code__.co_firstlineno + 1, _tryfinally.__code__.co_firstlineno + 2, _tryfinally.__code__.co_firstlineno + 4, + _tryfinally.__code__.co_firstlineno + 4, ) dis_tryfinallyconst = """\ -%3d RESUME 0 +%4d RESUME 0 -%3d NOP +%4d NOP -%3d NOP +%4d NOP -%3d LOAD_FAST 0 (b) - PUSH_NULL - CALL 0 - POP_TOP - RETURN_CONST 1 (1) - PUSH_EXC_INFO - LOAD_FAST 0 (b) - PUSH_NULL - CALL 0 - POP_TOP - RERAISE 0 - >> COPY 3 - POP_EXCEPT - RERAISE 1 +%4d LOAD_FAST 0 (b) + PUSH_NULL + CALL 0 + POP_TOP + RETURN_CONST 1 (1) + +None PUSH_EXC_INFO + +%4d LOAD_FAST 0 (b) + PUSH_NULL + CALL 0 + POP_TOP + RERAISE 0 + >> COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: 1 row """ % (_tryfinallyconst.__code__.co_firstlineno, _tryfinallyconst.__code__.co_firstlineno + 1, _tryfinallyconst.__code__.co_firstlineno + 2, _tryfinallyconst.__code__.co_firstlineno + 4, + _tryfinallyconst.__code__.co_firstlineno + 4, ) def _g(x): @@ -683,19 +703,19 @@ def foo(x): return foo dis_nested_0 = """\ - MAKE_CELL 0 (y) +None MAKE_CELL 0 (y) -%3d RESUME 0 +%4d RESUME 0 -%3d LOAD_FAST 0 (y) - BUILD_TUPLE 1 - LOAD_CONST 1 () - MAKE_FUNCTION - SET_FUNCTION_ATTRIBUTE 8 (closure) - STORE_FAST 1 (foo) +%4d LOAD_FAST 0 (y) + BUILD_TUPLE 1 + LOAD_CONST 1 () + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 8 (closure) + STORE_FAST 1 (foo) -%3d LOAD_FAST 1 (foo) - RETURN_VALUE +%4d LOAD_FAST 1 (foo) + RETURN_VALUE """ % (_h.__code__.co_firstlineno, _h.__code__.co_firstlineno + 1, __file__, @@ -705,22 +725,22 @@ def foo(x): dis_nested_1 = """%s Disassembly of : - COPY_FREE_VARS 1 - MAKE_CELL 0 (x) - -%3d RESUME 0 - -%3d LOAD_GLOBAL 1 (list + NULL) - LOAD_FAST 0 (x) - BUILD_TUPLE 1 - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) - MAKE_FUNCTION - SET_FUNCTION_ATTRIBUTE 8 (closure) - LOAD_DEREF 1 (y) - GET_ITER - CALL 0 - CALL 1 - RETURN_VALUE +None COPY_FREE_VARS 1 + MAKE_CELL 0 (x) + +%4d RESUME 0 + +%4d LOAD_GLOBAL 1 (list + NULL) + LOAD_FAST 0 (x) + BUILD_TUPLE 1 + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 8 (closure) + LOAD_DEREF 1 (y) + GET_ITER + CALL 0 + CALL 1 + RETURN_VALUE """ % (dis_nested_0, __file__, _h.__code__.co_firstlineno + 1, @@ -732,31 +752,35 @@ def foo(x): dis_nested_2 = """%s Disassembly of at 0x..., file "%s", line %d>: - COPY_FREE_VARS 1 +None COPY_FREE_VARS 1 -%3d RETURN_GENERATOR - POP_TOP - RESUME 0 - LOAD_FAST 0 (.0) - >> FOR_ITER 10 (to 34) - STORE_FAST 1 (z) - LOAD_DEREF 2 (x) - LOAD_FAST 1 (z) - BINARY_OP 0 (+) - YIELD_VALUE 1 - RESUME 1 - POP_TOP - JUMP_BACKWARD 12 (to 10) - >> END_FOR - RETURN_CONST 0 (None) - >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) - RERAISE 1 +%4d RETURN_GENERATOR + +None POP_TOP + +%4d RESUME 0 + LOAD_FAST 0 (.0) + >> FOR_ITER 10 (to 34) + STORE_FAST 1 (z) + LOAD_DEREF 2 (x) + LOAD_FAST 1 (z) + BINARY_OP 0 (+) + YIELD_VALUE 1 + RESUME 1 + POP_TOP + JUMP_BACKWARD 12 (to 10) + >> END_FOR + RETURN_CONST 0 (None) + +None >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 ExceptionTable: 1 row """ % (dis_nested_1, __file__, _h.__code__.co_firstlineno + 3, _h.__code__.co_firstlineno + 3, + _h.__code__.co_firstlineno + 3, ) def load_test(x, y=0): @@ -1616,197 +1640,197 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=74, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=69, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=70, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=74, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=69, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=70, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, is_jump_target=False, positions=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=104, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=74, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=104, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=74, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=104, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=147, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=104, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=147, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ - Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=31, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=114, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=125, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=24, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=146, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=67, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=125, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=42, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=67, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=66, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=20, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=31, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=114, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=True, line_number=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=True, line_number=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=True, line_number=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=125, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=False, line_number=8, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=24, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=True, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=False, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=False, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=False, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=146, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=True, line_number=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=True, line_number=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=False, line_number=12, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=12, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=True, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=False, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=67, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=False, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=False, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=True, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=False, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=False, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=False, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=True, line_number=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=True, line_number=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=False, line_number=16, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=False, line_number=16, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=False, line_number=16, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=125, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=True, line_number=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=True, line_number=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=False, line_number=19, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=42, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=True, line_number=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=True, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=False, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=67, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=False, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=True, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=True, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=False, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=True, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=True, line_number=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=True, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=66, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=True, line_number=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=20, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=22, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=False, line_number=22, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=22, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=True, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=True, line_number=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=True, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=28, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ - Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, is_jump_target=False), ] @@ -1916,7 +1940,12 @@ def roots(a, b, c): if show_caches or op != cache_opcode ] dis_positions = [ - instruction.positions + None if instruction.positions is None else ( + instruction.positions.lineno, + instruction.positions.end_lineno, + instruction.positions.col_offset, + instruction.positions.end_col_offset, + ) for instruction in dis.get_instructions( code, adaptive=adaptive, show_caches=show_caches ) @@ -1925,7 +1954,7 @@ def roots(a, b, c): def test_oparg_alias(self): instruction = Instruction(opname="NOP", opcode=dis.opmap["NOP"], arg=None, argval=None, - argrepr='', offset=10, start_offset=10, starts_line=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, positions=None) self.assertEqual(instruction.arg, instruction.oparg) @@ -1933,7 +1962,7 @@ def test_baseopname_and_baseopcode(self): # Standard instructions for name, code in dis.opmap.items(): instruction = Instruction(opname=name, opcode=code, arg=None, argval=None, argrepr='', offset=0, - start_offset=0, starts_line=1, is_jump_target=False, positions=None) + start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode self.assertIsNotNone(baseopname) @@ -1944,7 +1973,7 @@ def test_baseopname_and_baseopcode(self): # Specialized instructions for name in opcode._specialized_opmap: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', - offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) + offset=0, start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode self.assertIn(name, opcode._specializations[baseopname]) @@ -1953,25 +1982,25 @@ def test_baseopname_and_baseopcode(self): def test_jump_target(self): # Non-jump instructions should return None instruction = Instruction(opname="NOP", opcode=dis.opmap["NOP"], arg=None, argval=None, - argrepr='', offset=10, start_offset=10, starts_line=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, positions=None) self.assertIsNone(instruction.jump_target) delta = 100 instruction = Instruction(opname="JUMP_FORWARD", opcode=dis.opmap["JUMP_FORWARD"], arg=delta, argval=delta, - argrepr='', offset=10, start_offset=10, starts_line=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, positions=None) self.assertEqual(10 + 2 + 100*2, instruction.jump_target) # Test negative deltas instruction = Instruction(opname="JUMP_BACKWARD", opcode=dis.opmap["JUMP_BACKWARD"], arg=delta, argval=delta, - argrepr='', offset=200, start_offset=200, starts_line=1, is_jump_target=False, + argrepr='', offset=200, start_offset=200, starts_line=True, line_number=1, is_jump_target=False, positions=None) self.assertEqual(200 + 2 - 100*2 + 2*1, instruction.jump_target) # Make sure cache entries are handled instruction = Instruction(opname="SEND", opcode=dis.opmap["SEND"], arg=delta, argval=delta, - argrepr='', offset=10, start_offset=10, starts_line=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, positions=None) self.assertEqual(10 + 2 + 1*2 + 100*2, instruction.jump_target) diff --git a/Misc/ACKS b/Misc/ACKS index 70c4ea37adda75..e52208a41cc9f7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1191,6 +1191,7 @@ Andrew McNamara Caolan McNamara Jeff McNeil Craig McPheeters +Corvin McPherson Lambert Meertens Bill van Melle Lucas Prado Melo diff --git a/Misc/NEWS.d/next/Library/2023-08-16-21-20-55.gh-issue-107932.I7hFsp.rst b/Misc/NEWS.d/next/Library/2023-08-16-21-20-55.gh-issue-107932.I7hFsp.rst new file mode 100644 index 00000000000000..850cfcb6226d43 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-16-21-20-55.gh-issue-107932.I7hFsp.rst @@ -0,0 +1 @@ +Fix ``dis`` module to properly report and display bytecode that do not have source lines. From e59a95238b76f518e936b6e70da9207d923964db Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Aug 2023 11:13:59 +0200 Subject: [PATCH 326/598] gh-108444: Remove _PyLong_AsInt() function (#108461) * Update Parser/asdl_c.py to regenerate Python/Python-ast.c. * Remove _PyLong_AsInt() alias to PyLong_AsInt(). --- Include/cpython/longobject.h | 3 --- Parser/asdl_c.py | 2 +- Python/Python-ast.c | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index c96f35188c47fe..7401566636a104 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,9 +2,6 @@ # error "this header file must not be included directly" #endif -// Alias for backport compatibility -#define _PyLong_AsInt PyLong_AsInt - PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index bf144871bb7364..ca259c8cd1f3ba 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1074,7 +1074,7 @@ def visitModule(self, mod): return 1; } - i = _PyLong_AsInt(obj); + i = PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 60dd121d60b8d0..77b23f7c5edf23 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1099,7 +1099,7 @@ static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* o return 1; } - i = _PyLong_AsInt(obj); + i = PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; From 4ae3edf3008b70e20663143553a736d80ff3a501 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 25 Aug 2023 12:37:27 +0300 Subject: [PATCH 327/598] gh-108418: Speed up bigmem compression tests in dry mode (GH-108419) Only generate and compress small amount of random data in dry run. --- Lib/test/test_bz2.py | 4 ++-- Lib/test/test_lzma.py | 4 ++-- Lib/test/test_zlib.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index e4dd7fc2100b62..1f0b9adc3698b4 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -721,10 +721,10 @@ def testEOFError(self): @bigmemtest(size=_4G + 100, memuse=3.3) def testDecompress4G(self, size): # "Test BZ2Decompressor.decompress() with >4GiB input" - blocksize = 10 * 1024 * 1024 + blocksize = min(10 * 1024 * 1024, size) block = random.randbytes(blocksize) try: - data = block * (size // blocksize + 1) + data = block * ((size-1) // blocksize + 1) compressed = bz2.compress(data) bz2d = BZ2Decompressor() decompressed = bz2d.decompress(compressed) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index ac53bdda2f1747..13b200912f6abd 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -352,10 +352,10 @@ def test_compressor_bigmem(self, size): @bigmemtest(size=_4G + 100, memuse=3) def test_decompressor_bigmem(self, size): lzd = LZMADecompressor() - blocksize = 10 * 1024 * 1024 + blocksize = min(10 * 1024 * 1024, size) block = random.randbytes(blocksize) try: - input = block * (size // blocksize + 1) + input = block * ((size-1) // blocksize + 1) cdata = lzma.compress(input) ddata = lzd.decompress(cdata) self.assertEqual(ddata, input) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 2113757254c0ed..55306c63cd4e16 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -989,10 +989,10 @@ def testEOFError(self): @bigmemtest(size=_4G + 100, memuse=3.3) def testDecompress4G(self, size): # "Test zlib._ZlibDecompressor.decompress() with >4GiB input" - blocksize = 10 * 1024 * 1024 + blocksize = min(10 * 1024 * 1024, size) block = random.randbytes(blocksize) try: - data = block * (size // blocksize + 1) + data = block * ((size-1) // blocksize + 1) compressed = zlib.compress(data) zlibd = zlib._ZlibDecompressor() decompressed = zlibd.decompress(compressed) From 7f5b1a06612bf1454232ac634ad4d2c845f77b37 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 25 Aug 2023 11:27:36 +0100 Subject: [PATCH 328/598] Docs: Datamodel: Merge "Notes on using __slots__" with the parent section (#108400) --- Doc/reference/datamodel.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 229fa696c9142f..484dd9c49abf5a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1920,8 +1920,7 @@ Attribute lookup speed can be significantly improved as well. .. _datamodel-note-slots: -Notes on using *__slots__* -"""""""""""""""""""""""""" +Notes on using *__slots__*: * When inheriting from a class without *__slots__*, the :attr:`~object.__dict__` and From 2b7bff0655a4caf51cd1a9e5bf85b3b96dd031c9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:02:36 +0100 Subject: [PATCH 329/598] Datamodel: Add headings to the standard type hierarchy (#108146) Dedent content according to the new layout. --- Doc/reference/datamodel.rst | 2104 ++++++++++++++++++----------------- 1 file changed, 1104 insertions(+), 1000 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 484dd9c49abf5a..832a9150c16ad9 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -141,1095 +141,1199 @@ Some of the type descriptions below contain a paragraph listing 'special attributes.' These are attributes that provide access to the implementation and are not intended for general use. Their definition may change in the future. + None - .. index:: pair: object; None +---- + +.. index:: pair: object; None + +This type has a single value. There is a single object with this value. This +object is accessed through the built-in name ``None``. It is used to signify the +absence of a value in many situations, e.g., it is returned from functions that +don't explicitly return anything. Its truth value is false. - This type has a single value. There is a single object with this value. This - object is accessed through the built-in name ``None``. It is used to signify the - absence of a value in many situations, e.g., it is returned from functions that - don't explicitly return anything. Its truth value is false. NotImplemented - .. index:: pair: object; NotImplemented +-------------- + +.. index:: pair: object; NotImplemented - This type has a single value. There is a single object with this value. This - object is accessed through the built-in name ``NotImplemented``. Numeric methods - and rich comparison methods should return this value if they do not implement the - operation for the operands provided. (The interpreter will then try the - reflected operation, or some other fallback, depending on the operator.) It - should not be evaluated in a boolean context. +This type has a single value. There is a single object with this value. This +object is accessed through the built-in name ``NotImplemented``. Numeric methods +and rich comparison methods should return this value if they do not implement the +operation for the operands provided. (The interpreter will then try the +reflected operation, or some other fallback, depending on the operator.) It +should not be evaluated in a boolean context. - See - :ref:`implementing-the-arithmetic-operations` - for more details. +See +:ref:`implementing-the-arithmetic-operations` +for more details. - .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. +.. versionchanged:: 3.9 + Evaluating ``NotImplemented`` in a boolean context is deprecated. While + it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. + It will raise a :exc:`TypeError` in a future version of Python. Ellipsis - .. index:: - pair: object; Ellipsis - single: ...; ellipsis literal +-------- +.. index:: + pair: object; Ellipsis + single: ...; ellipsis literal + +This type has a single value. There is a single object with this value. This +object is accessed through the literal ``...`` or the built-in name +``Ellipsis``. Its truth value is true. - This type has a single value. There is a single object with this value. This - object is accessed through the literal ``...`` or the built-in name - ``Ellipsis``. Its truth value is true. :class:`numbers.Number` - .. index:: pair: object; numeric +----------------------- + +.. index:: pair: object; numeric + +These are created by numeric literals and returned as results by arithmetic +operators and arithmetic built-in functions. Numeric objects are immutable; +once created their value never changes. Python numbers are of course strongly +related to mathematical numbers, but subject to the limitations of numerical +representation in computers. + +The string representations of the numeric classes, computed by +:meth:`~object.__repr__` and :meth:`~object.__str__`, have the following +properties: + +* They are valid numeric literals which, when passed to their + class constructor, produce an object having the value of the + original numeric. + +* The representation is in base 10, when possible. + +* Leading zeros, possibly excepting a single zero before a + decimal point, are not shown. - These are created by numeric literals and returned as results by arithmetic - operators and arithmetic built-in functions. Numeric objects are immutable; - once created their value never changes. Python numbers are of course strongly - related to mathematical numbers, but subject to the limitations of numerical - representation in computers. +* Trailing zeros, possibly excepting a single zero after a + decimal point, are not shown. - The string representations of the numeric classes, computed by - :meth:`~object.__repr__` and :meth:`~object.__str__`, have the following - properties: +* A sign is shown only when the number is negative. - * They are valid numeric literals which, when passed to their - class constructor, produce an object having the value of the - original numeric. +Python distinguishes between integers, floating point numbers, and complex +numbers: - * The representation is in base 10, when possible. - * Leading zeros, possibly excepting a single zero before a - decimal point, are not shown. +:class:`numbers.Integral` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: pair: object; integer - * Trailing zeros, possibly excepting a single zero after a - decimal point, are not shown. +These represent elements from the mathematical set of integers (positive and +negative). - * A sign is shown only when the number is negative. +.. note:: + .. index:: pair: integer; representation - Python distinguishes between integers, floating point numbers, and complex - numbers: + The rules for integer representation are intended to give the most meaningful + interpretation of shift and mask operations involving negative integers. - :class:`numbers.Integral` - .. index:: pair: object; integer +There are two types of integers: - These represent elements from the mathematical set of integers (positive and - negative). +Integers (:class:`int`) + These represent numbers in an unlimited range, subject to available (virtual) + memory only. For the purpose of shift and mask operations, a binary + representation is assumed, and negative numbers are represented in a variant of + 2's complement which gives the illusion of an infinite string of sign bits + extending to the left. - There are two types of integers: +Booleans (:class:`bool`) + .. index:: + pair: object; Boolean + single: False + single: True - Integers (:class:`int`) - These represent numbers in an unlimited range, subject to available (virtual) - memory only. For the purpose of shift and mask operations, a binary - representation is assumed, and negative numbers are represented in a variant of - 2's complement which gives the illusion of an infinite string of sign bits - extending to the left. + These represent the truth values False and True. The two objects representing + the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a + subtype of the integer type, and Boolean values behave like the values 0 and 1, + respectively, in almost all contexts, the exception being that when converted to + a string, the strings ``"False"`` or ``"True"`` are returned, respectively. - Booleans (:class:`bool`) - .. index:: - pair: object; Boolean - single: False - single: True - These represent the truth values False and True. The two objects representing - the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a - subtype of the integer type, and Boolean values behave like the values 0 and 1, - respectively, in almost all contexts, the exception being that when converted to - a string, the strings ``"False"`` or ``"True"`` are returned, respectively. +:class:`numbers.Real` (:class:`float`) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; floating point + pair: floating point; number + pair: C; language + pair: Java; language - .. index:: pair: integer; representation +These represent machine-level double precision floating point numbers. You are +at the mercy of the underlying machine architecture (and C or Java +implementation) for the accepted range and handling of overflow. Python does not +support single-precision floating point numbers; the savings in processor and +memory usage that are usually the reason for using these are dwarfed by the +overhead of using objects in Python, so there is no reason to complicate the +language with two kinds of floating point numbers. - The rules for integer representation are intended to give the most meaningful - interpretation of shift and mask operations involving negative integers. - :class:`numbers.Real` (:class:`float`) - .. index:: - pair: object; floating point - pair: floating point; number - pair: C; language - pair: Java; language +:class:`numbers.Complex` (:class:`complex`) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - These represent machine-level double precision floating point numbers. You are - at the mercy of the underlying machine architecture (and C or Java - implementation) for the accepted range and handling of overflow. Python does not - support single-precision floating point numbers; the savings in processor and - memory usage that are usually the reason for using these are dwarfed by the - overhead of using objects in Python, so there is no reason to complicate the - language with two kinds of floating point numbers. +.. index:: + pair: object; complex + pair: complex; number - :class:`numbers.Complex` (:class:`complex`) - .. index:: - pair: object; complex - pair: complex; number +These represent complex numbers as a pair of machine-level double precision +floating point numbers. The same caveats apply as for floating point numbers. +The real and imaginary parts of a complex number ``z`` can be retrieved through +the read-only attributes ``z.real`` and ``z.imag``. - These represent complex numbers as a pair of machine-level double precision - floating point numbers. The same caveats apply as for floating point numbers. - The real and imaginary parts of a complex number ``z`` can be retrieved through - the read-only attributes ``z.real`` and ``z.imag``. Sequences +--------- + +.. index:: + pair: built-in function; len + pair: object; sequence + single: index operation + single: item selection + single: subscription + +These represent finite ordered sets indexed by non-negative numbers. The +built-in function :func:`len` returns the number of items of a sequence. When +the length of a sequence is *n*, the index set contains the numbers 0, 1, +..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. + +.. index:: single: slicing + +Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such +that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a +sequence of the same type. This implies that the index set is renumbered so +that it starts at 0. + +Some sequences also support "extended slicing" with a third "step" parameter: +``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* +``>=`` ``0`` and *i* ``<=`` *x* ``<`` *j*. + +Sequences are distinguished according to their mutability: + + +Immutable sequences +^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; immutable sequence + pair: object; immutable + +An object of an immutable sequence type cannot change once it is created. (If +the object contains references to other objects, these other objects may be +mutable and may be changed; however, the collection of objects directly +referenced by an immutable object cannot change.) + +The following types are immutable sequences: + +.. index:: + single: string; immutable sequences + +Strings .. index:: - pair: built-in function; len - pair: object; sequence - single: index operation - single: item selection - single: subscription - - These represent finite ordered sets indexed by non-negative numbers. The - built-in function :func:`len` returns the number of items of a sequence. When - the length of a sequence is *n*, the index set contains the numbers 0, 1, - ..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. - - .. index:: single: slicing - - Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such - that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a - sequence of the same type. This implies that the index set is renumbered so - that it starts at 0. - - Some sequences also support "extended slicing" with a third "step" parameter: - ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* - ``>=`` ``0`` and *i* ``<=`` *x* ``<`` *j*. - - Sequences are distinguished according to their mutability: - - Immutable sequences - .. index:: - pair: object; immutable sequence - pair: object; immutable - - An object of an immutable sequence type cannot change once it is created. (If - the object contains references to other objects, these other objects may be - mutable and may be changed; however, the collection of objects directly - referenced by an immutable object cannot change.) - - The following types are immutable sequences: - - .. index:: - single: string; immutable sequences - - Strings - .. index:: - pair: built-in function; chr - pair: built-in function; ord - single: character - single: integer - single: Unicode - - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:expr:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` - converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. - :meth:`str.encode` can be used to convert a :class:`str` to - :class:`bytes` using the given text encoding, and - :meth:`bytes.decode` can be used to achieve the opposite. - - Tuples - .. index:: - pair: object; tuple - pair: singleton; tuple - pair: empty; tuple - - The items of a tuple are arbitrary Python objects. Tuples of two or - more items are formed by comma-separated lists of expressions. A tuple - of one item (a 'singleton') can be formed by affixing a comma to an - expression (an expression by itself does not create a tuple, since - parentheses must be usable for grouping of expressions). An empty - tuple can be formed by an empty pair of parentheses. - - Bytes - .. index:: bytes, byte - - A bytes object is an immutable array. The items are 8-bit bytes, - represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in :func:`bytes()` constructor - can be used to create bytes objects. Also, bytes objects can be - decoded to strings via the :meth:`~bytes.decode` method. - - Mutable sequences - .. index:: - pair: object; mutable sequence - pair: object; mutable - pair: assignment; statement - single: subscription - single: slicing - - Mutable sequences can be changed after they are created. The subscription and - slicing notations can be used as the target of assignment and :keyword:`del` - (delete) statements. - - There are currently two intrinsic mutable sequence types: - - Lists - .. index:: pair: object; list - - The items of a list are arbitrary Python objects. Lists are formed by - placing a comma-separated list of expressions in square brackets. (Note - that there are no special cases needed to form lists of length 0 or 1.) - - Byte Arrays - .. index:: bytearray - - A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable - (and hence unhashable), byte arrays otherwise provide the same interface - and functionality as immutable :class:`bytes` objects. - - .. index:: pair: module; array - - The extension module :mod:`array` provides an additional example of a - mutable sequence type, as does the :mod:`collections` module. + pair: built-in function; chr + pair: built-in function; ord + single: character + single: integer + single: Unicode + + A string is a sequence of values that represent Unicode code points. + All the code points in the range ``U+0000 - U+10FFFF`` can be + represented in a string. Python doesn't have a :c:expr:`char` type; + instead, every code point in the string is represented as a string + object with length ``1``. The built-in function :func:`ord` + converts a code point from its string form to an integer in the + range ``0 - 10FFFF``; :func:`chr` converts an integer in the range + ``0 - 10FFFF`` to the corresponding length ``1`` string object. + :meth:`str.encode` can be used to convert a :class:`str` to + :class:`bytes` using the given text encoding, and + :meth:`bytes.decode` can be used to achieve the opposite. + +Tuples + .. index:: + pair: object; tuple + pair: singleton; tuple + pair: empty; tuple + + The items of a tuple are arbitrary Python objects. Tuples of two or + more items are formed by comma-separated lists of expressions. A tuple + of one item (a 'singleton') can be formed by affixing a comma to an + expression (an expression by itself does not create a tuple, since + parentheses must be usable for grouping of expressions). An empty + tuple can be formed by an empty pair of parentheses. + +Bytes + .. index:: bytes, byte + + A bytes object is an immutable array. The items are 8-bit bytes, + represented by integers in the range 0 <= x < 256. Bytes literals + (like ``b'abc'``) and the built-in :func:`bytes()` constructor + can be used to create bytes objects. Also, bytes objects can be + decoded to strings via the :meth:`~bytes.decode` method. + + +Mutable sequences +^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; mutable sequence + pair: object; mutable + pair: assignment; statement + single: subscription + single: slicing + +Mutable sequences can be changed after they are created. The subscription and +slicing notations can be used as the target of assignment and :keyword:`del` +(delete) statements. + +.. note:: + .. index:: pair: module; array + .. index:: pair: module; collections + + The :mod:`collections` and :mod:`array` module provide + additional examples of mutable sequence types. + +There are currently two intrinsic mutable sequence types: + +Lists + .. index:: pair: object; list + + The items of a list are arbitrary Python objects. Lists are formed by + placing a comma-separated list of expressions in square brackets. (Note + that there are no special cases needed to form lists of length 0 or 1.) + +Byte Arrays + .. index:: bytearray + + A bytearray object is a mutable array. They are created by the built-in + :func:`bytearray` constructor. Aside from being mutable + (and hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable :class:`bytes` objects. + Set types - .. index:: - pair: built-in function; len - pair: object; set type +--------- + +.. index:: + pair: built-in function; len + pair: object; set type + +These represent unordered, finite sets of unique, immutable objects. As such, +they cannot be indexed by any subscript. However, they can be iterated over, and +the built-in function :func:`len` returns the number of items in a set. Common +uses for sets are fast membership testing, removing duplicates from a sequence, +and computing mathematical operations such as intersection, union, difference, +and symmetric difference. - These represent unordered, finite sets of unique, immutable objects. As such, - they cannot be indexed by any subscript. However, they can be iterated over, and - the built-in function :func:`len` returns the number of items in a set. Common - uses for sets are fast membership testing, removing duplicates from a sequence, - and computing mathematical operations such as intersection, union, difference, - and symmetric difference. +For set elements, the same immutability rules apply as for dictionary keys. Note +that numeric types obey the normal rules for numeric comparison: if two numbers +compare equal (e.g., ``1`` and ``1.0``), only one of them can be contained in a +set. - For set elements, the same immutability rules apply as for dictionary keys. Note - that numeric types obey the normal rules for numeric comparison: if two numbers - compare equal (e.g., ``1`` and ``1.0``), only one of them can be contained in a - set. +There are currently two intrinsic set types: - There are currently two intrinsic set types: - Sets - .. index:: pair: object; set +Sets + .. index:: pair: object; set - These represent a mutable set. They are created by the built-in :func:`set` - constructor and can be modified afterwards by several methods, such as - :meth:`~set.add`. + These represent a mutable set. They are created by the built-in :func:`set` + constructor and can be modified afterwards by several methods, such as + :meth:`~set.add`. - Frozen sets - .. index:: pair: object; frozenset - These represent an immutable set. They are created by the built-in - :func:`frozenset` constructor. As a frozenset is immutable and - :term:`hashable`, it can be used again as an element of another set, or as - a dictionary key. +Frozen sets + .. index:: pair: object; frozenset + + These represent an immutable set. They are created by the built-in + :func:`frozenset` constructor. As a frozenset is immutable and + :term:`hashable`, it can be used again as an element of another set, or as + a dictionary key. + Mappings - .. index:: - pair: built-in function; len - single: subscription - pair: object; mapping - - These represent finite sets of objects indexed by arbitrary index sets. The - subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping - ``a``; this can be used in expressions and as the target of assignments or - :keyword:`del` statements. The built-in function :func:`len` returns the number - of items in a mapping. - - There is currently a single intrinsic mapping type: - - Dictionaries - .. index:: pair: object; dictionary - - These represent finite sets of objects indexed by nearly arbitrary values. The - only types of values not acceptable as keys are values containing lists or - dictionaries or other mutable types that are compared by value rather than by - object identity, the reason being that the efficient implementation of - dictionaries requires a key's hash value to remain constant. Numeric types used - for keys obey the normal rules for numeric comparison: if two numbers compare - equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index - the same dictionary entry. - - Dictionaries preserve insertion order, meaning that keys will be produced - in the same order they were added sequentially over the dictionary. - Replacing an existing key does not change the order, however removing a key - and re-inserting it will add it to the end instead of keeping its old place. - - Dictionaries are mutable; they can be created by the ``{...}`` notation (see - section :ref:`dict`). - - .. index:: - pair: module; dbm.ndbm - pair: module; dbm.gnu - - The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide - additional examples of mapping types, as does the :mod:`collections` - module. - - .. versionchanged:: 3.7 - Dictionaries did not preserve insertion order in versions of Python before 3.6. - In CPython 3.6, insertion order was preserved, but it was considered - an implementation detail at that time rather than a language guarantee. +-------- + +.. index:: + pair: built-in function; len + single: subscription + pair: object; mapping + +These represent finite sets of objects indexed by arbitrary index sets. The +subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping +``a``; this can be used in expressions and as the target of assignments or +:keyword:`del` statements. The built-in function :func:`len` returns the number +of items in a mapping. + +There is currently a single intrinsic mapping type: + + +Dictionaries +^^^^^^^^^^^^ + +.. index:: pair: object; dictionary + +These represent finite sets of objects indexed by nearly arbitrary values. The +only types of values not acceptable as keys are values containing lists or +dictionaries or other mutable types that are compared by value rather than by +object identity, the reason being that the efficient implementation of +dictionaries requires a key's hash value to remain constant. Numeric types used +for keys obey the normal rules for numeric comparison: if two numbers compare +equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index +the same dictionary entry. + +Dictionaries preserve insertion order, meaning that keys will be produced +in the same order they were added sequentially over the dictionary. +Replacing an existing key does not change the order, however removing a key +and re-inserting it will add it to the end instead of keeping its old place. + +Dictionaries are mutable; they can be created by the ``{...}`` notation (see +section :ref:`dict`). + +.. index:: + pair: module; dbm.ndbm + pair: module; dbm.gnu + +The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide +additional examples of mapping types, as does the :mod:`collections` +module. + +.. versionchanged:: 3.7 + Dictionaries did not preserve insertion order in versions of Python before 3.6. + In CPython 3.6, insertion order was preserved, but it was considered + an implementation detail at that time rather than a language guarantee. + Callable types - .. index:: - pair: object; callable - pair: function; call - single: invocation - pair: function; argument - - These are the types to which the function call operation (see section - :ref:`calls`) can be applied: - - User-defined functions - .. index:: - pair: user-defined; function - pair: object; function - pair: object; user-defined function - - A user-defined function object is created by a function definition (see - section :ref:`function`). It should be called with an argument list - containing the same number of items as the function's formal parameter - list. - - Special attributes: - - .. tabularcolumns:: |l|L|l| - - .. index:: - single: __doc__ (function attribute) - single: __name__ (function attribute) - single: __module__ (function attribute) - single: __dict__ (function attribute) - single: __defaults__ (function attribute) - single: __closure__ (function attribute) - single: __code__ (function attribute) - single: __globals__ (function attribute) - single: __annotations__ (function attribute) - single: __kwdefaults__ (function attribute) - single: __type_params__ (function attribute) - pair: global; namespace - - +-------------------------+-------------------------------+-----------+ - | Attribute | Meaning | | - +=========================+===============================+===========+ - | :attr:`__doc__` | The function's documentation | Writable | - | | string, or ``None`` if | | - | | unavailable; not inherited by | | - | | subclasses. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's name. | Writable | - | __name__` | | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's | Writable | - | __qualname__` | :term:`qualified name`. | | - | | | | - | | .. versionadded:: 3.3 | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__module__` | The name of the module the | Writable | - | | function was defined in, or | | - | | ``None`` if unavailable. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__defaults__` | A tuple containing default | Writable | - | | argument values for those | | - | | arguments that have defaults, | | - | | or ``None`` if no arguments | | - | | have a default value. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__code__` | The code object representing | Writable | - | | the compiled function body. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__globals__` | A reference to the dictionary | Read-only | - | | that holds the function's | | - | | global variables --- the | | - | | global namespace of the | | - | | module in which the function | | - | | was defined. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~object.__dict__`| The namespace supporting | Writable | - | | arbitrary function | | - | | attributes. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | - | | that contain bindings for the | | - | | function's free variables. | | - | | See below for information on | | - | | the ``cell_contents`` | | - | | attribute. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__annotations__` | A dict containing annotations | Writable | - | | of parameters. The keys of | | - | | the dict are the parameter | | - | | names, and ``'return'`` for | | - | | the return annotation, if | | - | | provided. For more | | - | | information on working with | | - | | this attribute, see | | - | | :ref:`annotations-howto`. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__kwdefaults__` | A dict containing defaults | Writable | - | | for keyword-only parameters. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__type_params__` | A tuple containing the | Writable | - | | :ref:`type parameters | | - | | ` of a | | - | | :ref:`generic function | | - | | `. | | - +-------------------------+-------------------------------+-----------+ - - Most of the attributes labelled "Writable" check the type of the assigned value. - - Function objects also support getting and setting arbitrary attributes, which - can be used, for example, to attach metadata to functions. Regular attribute - dot-notation is used to get and set such attributes. *Note that the current - implementation only supports function attributes on user-defined functions. - Function attributes on built-in functions may be supported in the future.* - - A cell object has the attribute ``cell_contents``. This can be used to get - the value of the cell, as well as set the value. - - Additional information about a function's definition can be retrieved from its - code object; see the description of internal types below. The - :data:`cell ` type can be accessed in the :mod:`types` - module. - - Instance methods - .. index:: - pair: object; method - pair: object; user-defined method - pair: user-defined; method - - An instance method object combines a class, a class instance and any - callable object (normally a user-defined function). - - .. index:: - single: __func__ (method attribute) - single: __self__ (method attribute) - single: __doc__ (method attribute) - single: __name__ (method attribute) - single: __module__ (method attribute) - - Special read-only attributes: :attr:`__self__` is the class instance object, - :attr:`__func__` is the function object; :attr:`__doc__` is the method's - documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the - method name (same as ``__func__.__name__``); :attr:`__module__` is the - name of the module the method was defined in, or ``None`` if unavailable. - - Methods also support accessing (but not setting) the arbitrary function - attributes on the underlying function object. - - User-defined method objects may be created when getting an attribute of a - class (perhaps via an instance of that class), if that attribute is a - user-defined function object or a class method object. - - When an instance method object is created by retrieving a user-defined - function object from a class via one of its instances, its - :attr:`__self__` attribute is the instance, and the method object is said - to be bound. The new method's :attr:`__func__` attribute is the original - function object. - - When an instance method object is created by retrieving a class method - object from a class or instance, its :attr:`__self__` attribute is the - class itself, and its :attr:`__func__` attribute is the function object - underlying the class method. - - When an instance method object is called, the underlying function - (:attr:`__func__`) is called, inserting the class instance - (:attr:`__self__`) in front of the argument list. For instance, when - :class:`C` is a class which contains a definition for a function - :meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is - equivalent to calling ``C.f(x, 1)``. - - When an instance method object is derived from a class method object, the - "class instance" stored in :attr:`__self__` will actually be the class - itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to - calling ``f(C,1)`` where ``f`` is the underlying function. - - Note that the transformation from function object to instance method - object happens each time the attribute is retrieved from the instance. In - some cases, a fruitful optimization is to assign the attribute to a local - variable and call that local variable. Also notice that this - transformation only happens for user-defined functions; other callable - objects (and all non-callable objects) are retrieved without - transformation. It is also important to note that user-defined functions - which are attributes of a class instance are not converted to bound - methods; this *only* happens when the function is an attribute of the - class. - - Generator functions - .. index:: - single: generator; function - single: generator; iterator - - A function or method which uses the :keyword:`yield` statement (see section - :ref:`yield`) is called a :dfn:`generator function`. Such a function, when - called, always returns an :term:`iterator` object which can be used to - execute the body of the function: calling the iterator's - :meth:`iterator.__next__` method will cause the function to execute until - it provides a value using the :keyword:`!yield` statement. When the - function executes a :keyword:`return` statement or falls off the end, a - :exc:`StopIteration` exception is raised and the iterator will have - reached the end of the set of values to be returned. - - Coroutine functions - .. index:: - single: coroutine; function - - A function or method which is defined using :keyword:`async def` is called - a :dfn:`coroutine function`. Such a function, when called, returns a - :term:`coroutine` object. It may contain :keyword:`await` expressions, - as well as :keyword:`async with` and :keyword:`async for` statements. See - also the :ref:`coroutine-objects` section. - - Asynchronous generator functions - .. index:: - single: asynchronous generator; function - single: asynchronous generator; asynchronous iterator - - A function or method which is defined using :keyword:`async def` and - which uses the :keyword:`yield` statement is called a - :dfn:`asynchronous generator function`. Such a function, when called, - returns an :term:`asynchronous iterator` object which can be used in an - :keyword:`async for` statement to execute the body of the function. - - Calling the asynchronous iterator's - :meth:`aiterator.__anext__ ` method - will return an :term:`awaitable` which when awaited - will execute until it provides a value using the :keyword:`yield` - expression. When the function executes an empty :keyword:`return` - statement or falls off the end, a :exc:`StopAsyncIteration` exception - is raised and the asynchronous iterator will have reached the end of - the set of values to be yielded. - - Built-in functions - .. index:: - pair: object; built-in function - pair: object; function - pair: C; language - - A built-in function object is a wrapper around a C function. Examples of - built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a - standard built-in module). The number and type of the arguments are - determined by the C function. Special read-only attributes: - :attr:`__doc__` is the function's documentation string, or ``None`` if - unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is - set to ``None`` (but see the next item); :attr:`__module__` is the name of - the module the function was defined in or ``None`` if unavailable. - - Built-in methods - .. index:: - pair: object; built-in method - pair: object; method - pair: built-in; method - - This is really a different disguise of a built-in function, this time containing - an object passed to the C function as an implicit extra argument. An example of - a built-in method is ``alist.append()``, assuming *alist* is a list object. In - this case, the special read-only attribute :attr:`__self__` is set to the object - denoted by *alist*. - - Classes - Classes are callable. These objects normally act as factories for new - instances of themselves, but variations are possible for class types that - override :meth:`~object.__new__`. The arguments of the call are passed to - :meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to - initialize the new instance. - - Class Instances - Instances of arbitrary classes can be made callable by defining a - :meth:`~object.__call__` method in their class. +-------------- + +.. index:: + pair: object; callable + pair: function; call + single: invocation + pair: function; argument + +These are the types to which the function call operation (see section +:ref:`calls`) can be applied: + + +User-defined functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: user-defined; function + pair: object; function + pair: object; user-defined function + +A user-defined function object is created by a function definition (see +section :ref:`function`). It should be called with an argument list +containing the same number of items as the function's formal parameter +list. + +Special attributes: + +.. tabularcolumns:: |l|L|l| + +.. index:: + single: __doc__ (function attribute) + single: __name__ (function attribute) + single: __module__ (function attribute) + single: __dict__ (function attribute) + single: __defaults__ (function attribute) + single: __closure__ (function attribute) + single: __code__ (function attribute) + single: __globals__ (function attribute) + single: __annotations__ (function attribute) + single: __kwdefaults__ (function attribute) + single: __type_params__ (function attribute) + pair: global; namespace + ++-------------------------+-------------------------------+-----------+ +| Attribute | Meaning | | ++=========================+===============================+===========+ +| :attr:`__doc__` | The function's documentation | Writable | +| | string, or ``None`` if | | +| | unavailable; not inherited by | | +| | subclasses. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`~definition.\ | The function's name. | Writable | +| __name__` | | | ++-------------------------+-------------------------------+-----------+ +| :attr:`~definition.\ | The function's | Writable | +| __qualname__` | :term:`qualified name`. | | +| | | | +| | .. versionadded:: 3.3 | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__module__` | The name of the module the | Writable | +| | function was defined in, or | | +| | ``None`` if unavailable. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__defaults__` | A tuple containing default | Writable | +| | argument values for those | | +| | arguments that have defaults, | | +| | or ``None`` if no arguments | | +| | have a default value. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__code__` | The code object representing | Writable | +| | the compiled function body. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__globals__` | A reference to the dictionary | Read-only | +| | that holds the function's | | +| | global variables --- the | | +| | global namespace of the | | +| | module in which the function | | +| | was defined. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`~object.__dict__`| The namespace supporting | Writable | +| | arbitrary function | | +| | attributes. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | +| | that contain bindings for the | | +| | function's free variables. | | +| | See below for information on | | +| | the ``cell_contents`` | | +| | attribute. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__annotations__` | A dict containing annotations | Writable | +| | of parameters. The keys of | | +| | the dict are the parameter | | +| | names, and ``'return'`` for | | +| | the return annotation, if | | +| | provided. For more | | +| | information on working with | | +| | this attribute, see | | +| | :ref:`annotations-howto`. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__kwdefaults__` | A dict containing defaults | Writable | +| | for keyword-only parameters. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__type_params__` | A tuple containing the | Writable | +| | :ref:`type parameters | | +| | ` of a | | +| | :ref:`generic function | | +| | `. | | ++-------------------------+-------------------------------+-----------+ + +Most of the attributes labelled "Writable" check the type of the assigned value. + +Function objects also support getting and setting arbitrary attributes, which +can be used, for example, to attach metadata to functions. Regular attribute +dot-notation is used to get and set such attributes. *Note that the current +implementation only supports function attributes on user-defined functions. +Function attributes on built-in functions may be supported in the future.* + +A cell object has the attribute ``cell_contents``. This can be used to get +the value of the cell, as well as set the value. + +Additional information about a function's definition can be retrieved from its +code object; see the description of internal types below. The +:data:`cell ` type can be accessed in the :mod:`types` +module. + + +Instance methods +^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; method + pair: object; user-defined method + pair: user-defined; method + +An instance method object combines a class, a class instance and any +callable object (normally a user-defined function). + +.. index:: + single: __func__ (method attribute) + single: __self__ (method attribute) + single: __doc__ (method attribute) + single: __name__ (method attribute) + single: __module__ (method attribute) + +Special read-only attributes: :attr:`__self__` is the class instance object, +:attr:`__func__` is the function object; :attr:`__doc__` is the method's +documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the +method name (same as ``__func__.__name__``); :attr:`__module__` is the +name of the module the method was defined in, or ``None`` if unavailable. + +Methods also support accessing (but not setting) the arbitrary function +attributes on the underlying function object. + +User-defined method objects may be created when getting an attribute of a +class (perhaps via an instance of that class), if that attribute is a +user-defined function object or a class method object. + +When an instance method object is created by retrieving a user-defined +function object from a class via one of its instances, its +:attr:`__self__` attribute is the instance, and the method object is said +to be bound. The new method's :attr:`__func__` attribute is the original +function object. + +When an instance method object is created by retrieving a class method +object from a class or instance, its :attr:`__self__` attribute is the +class itself, and its :attr:`__func__` attribute is the function object +underlying the class method. + +When an instance method object is called, the underlying function +(:attr:`__func__`) is called, inserting the class instance +(:attr:`__self__`) in front of the argument list. For instance, when +:class:`C` is a class which contains a definition for a function +:meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is +equivalent to calling ``C.f(x, 1)``. + +When an instance method object is derived from a class method object, the +"class instance" stored in :attr:`__self__` will actually be the class +itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to +calling ``f(C,1)`` where ``f`` is the underlying function. + +Note that the transformation from function object to instance method +object happens each time the attribute is retrieved from the instance. In +some cases, a fruitful optimization is to assign the attribute to a local +variable and call that local variable. Also notice that this +transformation only happens for user-defined functions; other callable +objects (and all non-callable objects) are retrieved without +transformation. It is also important to note that user-defined functions +which are attributes of a class instance are not converted to bound +methods; this *only* happens when the function is an attribute of the +class. + + +Generator functions +^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: generator; function + single: generator; iterator + +A function or method which uses the :keyword:`yield` statement (see section +:ref:`yield`) is called a :dfn:`generator function`. Such a function, when +called, always returns an :term:`iterator` object which can be used to +execute the body of the function: calling the iterator's +:meth:`iterator.__next__` method will cause the function to execute until +it provides a value using the :keyword:`!yield` statement. When the +function executes a :keyword:`return` statement or falls off the end, a +:exc:`StopIteration` exception is raised and the iterator will have +reached the end of the set of values to be returned. + + +Coroutine functions +^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: coroutine; function + +A function or method which is defined using :keyword:`async def` is called +a :dfn:`coroutine function`. Such a function, when called, returns a +:term:`coroutine` object. It may contain :keyword:`await` expressions, +as well as :keyword:`async with` and :keyword:`async for` statements. See +also the :ref:`coroutine-objects` section. + + +Asynchronous generator functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: asynchronous generator; function + single: asynchronous generator; asynchronous iterator + +A function or method which is defined using :keyword:`async def` and +which uses the :keyword:`yield` statement is called a +:dfn:`asynchronous generator function`. Such a function, when called, +returns an :term:`asynchronous iterator` object which can be used in an +:keyword:`async for` statement to execute the body of the function. + +Calling the asynchronous iterator's +:meth:`aiterator.__anext__ ` method +will return an :term:`awaitable` which when awaited +will execute until it provides a value using the :keyword:`yield` +expression. When the function executes an empty :keyword:`return` +statement or falls off the end, a :exc:`StopAsyncIteration` exception +is raised and the asynchronous iterator will have reached the end of +the set of values to be yielded. + + +Built-in functions +^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; built-in function + pair: object; function + pair: C; language + +A built-in function object is a wrapper around a C function. Examples of +built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a +standard built-in module). The number and type of the arguments are +determined by the C function. Special read-only attributes: +:attr:`__doc__` is the function's documentation string, or ``None`` if +unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is +set to ``None`` (but see the next item); :attr:`__module__` is the name of +the module the function was defined in or ``None`` if unavailable. + + +Built-in methods +^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; built-in method + pair: object; method + pair: built-in; method + +This is really a different disguise of a built-in function, this time containing +an object passed to the C function as an implicit extra argument. An example of +a built-in method is ``alist.append()``, assuming *alist* is a list object. In +this case, the special read-only attribute :attr:`__self__` is set to the object +denoted by *alist*. + + +Classes +^^^^^^^ + +Classes are callable. These objects normally act as factories for new +instances of themselves, but variations are possible for class types that +override :meth:`~object.__new__`. The arguments of the call are passed to +:meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to +initialize the new instance. + + +Class Instances +^^^^^^^^^^^^^^^ + +Instances of arbitrary classes can be made callable by defining a +:meth:`~object.__call__` method in their class. Modules - .. index:: - pair: statement; import - pair: object; module - - Modules are a basic organizational unit of Python code, and are created by - the :ref:`import system ` as invoked either by the - :keyword:`import` statement, or by calling - functions such as :func:`importlib.import_module` and built-in - :func:`__import__`. A module object has a namespace implemented by a - dictionary object (this is the dictionary referenced by the ``__globals__`` - attribute of functions defined in the module). Attribute references are - translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to - ``m.__dict__["x"]``. A module object does not contain the code object used - to initialize the module (since it isn't needed once the initialization is - done). - - Attribute assignment updates the module's namespace dictionary, e.g., - ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. +------- - .. index:: - single: __name__ (module attribute) - single: __doc__ (module attribute) - single: __file__ (module attribute) - single: __annotations__ (module attribute) - pair: module; namespace +.. index:: + pair: statement; import + pair: object; module + +Modules are a basic organizational unit of Python code, and are created by +the :ref:`import system ` as invoked either by the +:keyword:`import` statement, or by calling +functions such as :func:`importlib.import_module` and built-in +:func:`__import__`. A module object has a namespace implemented by a +dictionary object (this is the dictionary referenced by the ``__globals__`` +attribute of functions defined in the module). Attribute references are +translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to +``m.__dict__["x"]``. A module object does not contain the code object used +to initialize the module (since it isn't needed once the initialization is +done). + +Attribute assignment updates the module's namespace dictionary, e.g., +``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. - Predefined (writable) attributes: +.. index:: + single: __name__ (module attribute) + single: __doc__ (module attribute) + single: __file__ (module attribute) + single: __annotations__ (module attribute) + pair: module; namespace - :attr:`__name__` - The module's name. +Predefined (writable) attributes: - :attr:`__doc__` - The module's documentation string, or ``None`` if - unavailable. + :attr:`__name__` + The module's name. - :attr:`__file__` - The pathname of the file from which the - module was loaded, if it was loaded from a file. - The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter. For extension modules - loaded dynamically from a shared library, it's the pathname of the shared - library file. + :attr:`__doc__` + The module's documentation string, or ``None`` if + unavailable. - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` collected during - module body execution. For best practices on working - with :attr:`__annotations__`, please see :ref:`annotations-howto`. + :attr:`__file__` + The pathname of the file from which the + module was loaded, if it was loaded from a file. + The :attr:`__file__` + attribute may be missing for certain types of modules, such as C modules + that are statically linked into the interpreter. For extension modules + loaded dynamically from a shared library, it's the pathname of the shared + library file. - .. index:: single: __dict__ (module attribute) + :attr:`__annotations__` + A dictionary containing + :term:`variable annotations ` collected during + module body execution. For best practices on working + with :attr:`__annotations__`, please see :ref:`annotations-howto`. - Special read-only attribute: :attr:`~object.__dict__` is the module's - namespace as a dictionary object. +.. index:: single: __dict__ (module attribute) - .. impl-detail:: +Special read-only attribute: :attr:`~object.__dict__` is the module's +namespace as a dictionary object. + +.. impl-detail:: + + Because of the way CPython clears module dictionaries, the module + dictionary will be cleared when the module falls out of scope even if the + dictionary still has live references. To avoid this, copy the dictionary + or keep the module around while using its dictionary directly. - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. Custom classes - Custom class types are typically created by class definitions (see section - :ref:`class`). A class has a namespace implemented by a dictionary object. - Class attribute references are translated to lookups in this dictionary, e.g., - ``C.x`` is translated to ``C.__dict__["x"]`` (although there are a number of - hooks which allow for other means of locating attributes). When the attribute - name is not found there, the attribute search continues in the base classes. - This search of the base classes uses the C3 method resolution order which - behaves correctly even in the presence of 'diamond' inheritance structures - where there are multiple inheritance paths leading back to a common ancestor. - Additional details on the C3 MRO used by Python can be found in the - documentation accompanying the 2.3 release at - https://www.python.org/download/releases/2.3/mro/. - - .. XXX: Could we add that MRO doc as an appendix to the language ref? +-------------- + +Custom class types are typically created by class definitions (see section +:ref:`class`). A class has a namespace implemented by a dictionary object. +Class attribute references are translated to lookups in this dictionary, e.g., +``C.x`` is translated to ``C.__dict__["x"]`` (although there are a number of +hooks which allow for other means of locating attributes). When the attribute +name is not found there, the attribute search continues in the base classes. +This search of the base classes uses the C3 method resolution order which +behaves correctly even in the presence of 'diamond' inheritance structures +where there are multiple inheritance paths leading back to a common ancestor. +Additional details on the C3 MRO used by Python can be found in the +documentation accompanying the 2.3 release at +https://www.python.org/download/releases/2.3/mro/. + +.. XXX: Could we add that MRO doc as an appendix to the language ref? - .. index:: - pair: object; class - pair: object; class instance - pair: object; instance - pair: class object; call - single: container - pair: object; dictionary - pair: class; attribute +.. index:: + pair: object; class + pair: object; class instance + pair: object; instance + pair: class object; call + single: container + pair: object; dictionary + pair: class; attribute - When a class attribute reference (for class :class:`C`, say) would yield a - class method object, it is transformed into an instance method object whose - :attr:`__self__` attribute is :class:`C`. When it would yield a static - method object, it is transformed into the object wrapped by the static method - object. See section :ref:`descriptors` for another way in which attributes - retrieved from a class may differ from those actually contained in its - :attr:`~object.__dict__`. +When a class attribute reference (for class :class:`C`, say) would yield a +class method object, it is transformed into an instance method object whose +:attr:`__self__` attribute is :class:`C`. When it would yield a static +method object, it is transformed into the object wrapped by the static method +object. See section :ref:`descriptors` for another way in which attributes +retrieved from a class may differ from those actually contained in its +:attr:`~object.__dict__`. - .. index:: triple: class; attribute; assignment +.. index:: triple: class; attribute; assignment - Class attribute assignments update the class's dictionary, never the dictionary - of a base class. +Class attribute assignments update the class's dictionary, never the dictionary +of a base class. - .. index:: pair: class object; call +.. index:: pair: class object; call - A class object can be called (see above) to yield a class instance (see below). +A class object can be called (see above) to yield a class instance (see below). - .. index:: - single: __name__ (class attribute) - single: __module__ (class attribute) - single: __dict__ (class attribute) - single: __bases__ (class attribute) - single: __doc__ (class attribute) - single: __annotations__ (class attribute) - single: __type_params__ (class attribute) +.. index:: + single: __name__ (class attribute) + single: __module__ (class attribute) + single: __dict__ (class attribute) + single: __bases__ (class attribute) + single: __doc__ (class attribute) + single: __annotations__ (class attribute) + single: __type_params__ (class attribute) + +Special attributes: - Special attributes: + :attr:`~definition.__name__` + The class name. - :attr:`~definition.__name__` - The class name. + :attr:`__module__` + The name of the module in which the class was defined. - :attr:`__module__` - The name of the module in which the class was defined. + :attr:`~object.__dict__` + The dictionary containing the class's namespace. - :attr:`~object.__dict__` - The dictionary containing the class's namespace. + :attr:`~class.__bases__` + A tuple containing the base classes, in the order of + their occurrence in the base class list. - :attr:`~class.__bases__` - A tuple containing the base classes, in the order of - their occurrence in the base class list. + :attr:`__doc__` + The class's documentation string, or ``None`` if undefined. - :attr:`__doc__` - The class's documentation string, or ``None`` if undefined. + :attr:`__annotations__` + A dictionary containing + :term:`variable annotations ` + collected during class body execution. For best practices on + working with :attr:`__annotations__`, please see + :ref:`annotations-howto`. - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` - collected during class body execution. For best practices on - working with :attr:`__annotations__`, please see - :ref:`annotations-howto`. + :attr:`__type_params__` + A tuple containing the :ref:`type parameters ` of + a :ref:`generic class `. - :attr:`__type_params__` - A tuple containing the :ref:`type parameters ` of - a :ref:`generic class `. Class instances - .. index:: - pair: object; class instance - pair: object; instance - pair: class; instance - pair: class instance; attribute - - A class instance is created by calling a class object (see above). A class - instance has a namespace implemented as a dictionary which is the first place - in which attribute references are searched. When an attribute is not found - there, and the instance's class has an attribute by that name, the search - continues with the class attributes. If a class attribute is found that is a - user-defined function object, it is transformed into an instance method - object whose :attr:`__self__` attribute is the instance. Static method and - class method objects are also transformed; see above under "Classes". See - section :ref:`descriptors` for another way in which attributes of a class - retrieved via its instances may differ from the objects actually stored in - the class's :attr:`~object.__dict__`. If no class attribute is found, and the - object's class has a :meth:`~object.__getattr__` method, that is called to satisfy - the lookup. - - .. index:: triple: class instance; attribute; assignment - - Attribute assignments and deletions update the instance's dictionary, never a - class's dictionary. If the class has a :meth:`~object.__setattr__` or - :meth:`~object.__delattr__` method, this is called instead of updating the instance - dictionary directly. +--------------- - .. index:: - pair: object; numeric - pair: object; sequence - pair: object; mapping +.. index:: + pair: object; class instance + pair: object; instance + pair: class; instance + pair: class instance; attribute + +A class instance is created by calling a class object (see above). A class +instance has a namespace implemented as a dictionary which is the first place +in which attribute references are searched. When an attribute is not found +there, and the instance's class has an attribute by that name, the search +continues with the class attributes. If a class attribute is found that is a +user-defined function object, it is transformed into an instance method +object whose :attr:`__self__` attribute is the instance. Static method and +class method objects are also transformed; see above under "Classes". See +section :ref:`descriptors` for another way in which attributes of a class +retrieved via its instances may differ from the objects actually stored in +the class's :attr:`~object.__dict__`. If no class attribute is found, and the +object's class has a :meth:`~object.__getattr__` method, that is called to satisfy +the lookup. + +.. index:: triple: class instance; attribute; assignment + +Attribute assignments and deletions update the instance's dictionary, never a +class's dictionary. If the class has a :meth:`~object.__setattr__` or +:meth:`~object.__delattr__` method, this is called instead of updating the instance +dictionary directly. - Class instances can pretend to be numbers, sequences, or mappings if they have - methods with certain special names. See section :ref:`specialnames`. +.. index:: + pair: object; numeric + pair: object; sequence + pair: object; mapping - .. index:: - single: __dict__ (instance attribute) - single: __class__ (instance attribute) +Class instances can pretend to be numbers, sequences, or mappings if they have +methods with certain special names. See section :ref:`specialnames`. + +.. index:: + single: __dict__ (instance attribute) + single: __class__ (instance attribute) + +Special attributes: :attr:`~object.__dict__` is the attribute dictionary; +:attr:`~instance.__class__` is the instance's class. - Special attributes: :attr:`~object.__dict__` is the attribute dictionary; - :attr:`~instance.__class__` is the instance's class. I/O objects (also known as file objects) - .. index:: - pair: built-in function; open - pair: module; io - single: popen() (in module os) - single: makefile() (socket method) - single: sys.stdin - single: sys.stdout - single: sys.stderr - single: stdio - single: stdin (in module sys) - single: stdout (in module sys) - single: stderr (in module sys) - - A :term:`file object` represents an open file. Various shortcuts are - available to create file objects: the :func:`open` built-in function, and - also :func:`os.popen`, :func:`os.fdopen`, and the - :meth:`~socket.socket.makefile` method of socket objects (and perhaps by - other functions or methods provided by extension modules). - - The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are - initialized to file objects corresponding to the interpreter's standard - input, output and error streams; they are all open in text mode and - therefore follow the interface defined by the :class:`io.TextIOBase` - abstract class. +---------------------------------------- + +.. index:: + pair: built-in function; open + pair: module; io + single: popen() (in module os) + single: makefile() (socket method) + single: sys.stdin + single: sys.stdout + single: sys.stderr + single: stdio + single: stdin (in module sys) + single: stdout (in module sys) + single: stderr (in module sys) + +A :term:`file object` represents an open file. Various shortcuts are +available to create file objects: the :func:`open` built-in function, and +also :func:`os.popen`, :func:`os.fdopen`, and the +:meth:`~socket.socket.makefile` method of socket objects (and perhaps by +other functions or methods provided by extension modules). + +The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are +initialized to file objects corresponding to the interpreter's standard +input, output and error streams; they are all open in text mode and +therefore follow the interface defined by the :class:`io.TextIOBase` +abstract class. + Internal types - .. index:: - single: internal type - single: types, internal - - A few types used internally by the interpreter are exposed to the user. Their - definitions may change with future versions of the interpreter, but they are - mentioned here for completeness. - - .. index:: bytecode, object; code, code object - - Code objects - Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. - The difference between a code object and a function object is that the function - object contains an explicit reference to the function's globals (the module in - which it was defined), while a code object contains no context; also the default - argument values are stored in the function object, not in the code object - (because they represent values calculated at run-time). Unlike function - objects, code objects are immutable and contain no references (directly or - indirectly) to mutable objects. - - .. index:: - single: co_argcount (code object attribute) - single: co_posonlyargcount (code object attribute) - single: co_kwonlyargcount (code object attribute) - single: co_code (code object attribute) - single: co_consts (code object attribute) - single: co_filename (code object attribute) - single: co_firstlineno (code object attribute) - single: co_flags (code object attribute) - single: co_lnotab (code object attribute) - single: co_name (code object attribute) - single: co_names (code object attribute) - single: co_nlocals (code object attribute) - single: co_stacksize (code object attribute) - single: co_varnames (code object attribute) - single: co_cellvars (code object attribute) - single: co_freevars (code object attribute) - single: co_qualname (code object attribute) - - Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_qualname` gives the fully qualified function name; - :attr:`co_argcount` is the total number of positional arguments - (including positional-only arguments and arguments with default values); - :attr:`co_posonlyargcount` is the number of positional-only arguments - (including arguments with default values); :attr:`co_kwonlyargcount` is - the number of keyword-only arguments (including arguments with default - values); :attr:`co_nlocals` is the number of local variables used by the - function (including arguments); :attr:`co_varnames` is a tuple containing - the names of the local variables (starting with the argument names); - :attr:`co_cellvars` is a tuple containing the names of local variables - that are referenced by nested functions; :attr:`co_freevars` is a tuple - containing the names of free variables; :attr:`co_code` is a string - representing the sequence of bytecode instructions; :attr:`co_consts` is - a tuple containing the literals used by the bytecode; :attr:`co_names` is - a tuple containing the names used by the bytecode; :attr:`co_filename` is - the filename from which the code was compiled; :attr:`co_firstlineno` is - the first line number of the function; :attr:`co_lnotab` is a string - encoding the mapping from bytecode offsets to line numbers (for details - see the source code of the interpreter, is deprecated since 3.12 - and may be removed in 3.14); :attr:`co_stacksize` is the - required stack size; :attr:`co_flags` is an integer encoding a number - of flags for the interpreter. - - .. index:: pair: object; generator - - The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if - the function uses the ``*arguments`` syntax to accept an arbitrary number of - positional arguments; bit ``0x08`` is set if the function uses the - ``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set - if the function is a generator. - - Future feature declarations (``from __future__ import division``) also use bits - in :attr:`co_flags` to indicate whether a code object was compiled with a - particular feature enabled: bit ``0x2000`` is set if the function was compiled - with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier - versions of Python. - - Other bits in :attr:`co_flags` are reserved for internal use. - - .. index:: single: documentation string - - If a code object represents a function, the first item in :attr:`co_consts` is - the documentation string of the function, or ``None`` if undefined. - - .. method:: codeobject.co_positions() - - Returns an iterable over the source code positions of each bytecode - instruction in the code object. - - The iterator returns tuples containing the ``(start_line, end_line, - start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. - Column information is 0-indexed utf-8 byte offsets on the given source - line. - - This positional information can be missing. A non-exhaustive lists of - cases where this may happen: - - - Running the interpreter with :option:`-X` ``no_debug_ranges``. - - Loading a pyc file compiled while using :option:`-X` ``no_debug_ranges``. - - Position tuples corresponding to artificial instructions. - - Line and column numbers that can't be represented due to - implementation specific limitations. - - When this occurs, some or all of the tuple elements can be - :const:`None`. - - .. versionadded:: 3.11 - - .. note:: - This feature requires storing column positions in code objects which may - result in a small increase of disk usage of compiled Python files or - interpreter memory usage. To avoid storing the extra information and/or - deactivate printing the extra traceback information, the - :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` - environment variable can be used. - - .. _frame-objects: - - Frame objects - .. index:: pair: object; frame - - Frame objects represent execution frames. They may occur in traceback objects - (see below), and are also passed to registered trace functions. - - .. index:: - single: f_back (frame attribute) - single: f_code (frame attribute) - single: f_globals (frame attribute) - single: f_locals (frame attribute) - single: f_lasti (frame attribute) - single: f_builtins (frame attribute) - - Special read-only attributes: :attr:`f_back` is to the previous stack frame - (towards the caller), or ``None`` if this is the bottom stack frame; - :attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` - is the dictionary used to look up local variables; :attr:`f_globals` is used for - global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; - :attr:`f_lasti` gives the precise instruction (this is an index into the - bytecode string of the code object). - - Accessing ``f_code`` raises an :ref:`auditing event ` - ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. - - .. index:: - single: f_trace (frame attribute) - single: f_trace_lines (frame attribute) - single: f_trace_opcodes (frame attribute) - single: f_lineno (frame attribute) - - Special writable attributes: :attr:`f_trace`, if not ``None``, is a function - called for various events during code execution (this is used by the debugger). - Normally an event is triggered for each new source line - this can be - disabled by setting :attr:`f_trace_lines` to :const:`False`. - - Implementations *may* allow per-opcode events to be requested by setting - :attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to - undefined interpreter behaviour if exceptions raised by the trace - function escape to the function being traced. - - :attr:`f_lineno` is the current line number of the frame --- writing to this - from within a trace function jumps to the given line (only for the bottom-most - frame). A debugger can implement a Jump command (aka Set Next Statement) - by writing to f_lineno. - - Frame objects support one method: - - .. method:: frame.clear() - - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator - is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). - - :exc:`RuntimeError` is raised if the frame is currently executing. - - .. versionadded:: 3.4 - - .. _traceback-objects: - - Traceback objects - .. index:: - pair: object; traceback - pair: stack; trace - pair: exception; handler - pair: execution; stack - single: exc_info (in module sys) - single: last_traceback (in module sys) - single: sys.exc_info - single: sys.exception - single: sys.last_traceback - - Traceback objects represent a stack trace of an exception. A traceback object - is implicitly created when an exception occurs, and may also be explicitly - created by calling :class:`types.TracebackType`. - - For implicitly created tracebacks, when the search for an exception handler - unwinds the execution stack, at each unwound level a traceback object is - inserted in front of the current traceback. When an exception handler is - entered, the stack trace is made available to the program. (See section - :ref:`try`.) It is accessible as the third item of the - tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute - of the caught exception. - - When the program contains no suitable - handler, the stack trace is written (nicely formatted) to the standard error - stream; if the interpreter is interactive, it is also made available to the user - as ``sys.last_traceback``. - - For explicitly created tracebacks, it is up to the creator of the traceback - to determine how the ``tb_next`` attributes should be linked to form a - full stack trace. - - .. index:: - single: tb_frame (traceback attribute) - single: tb_lineno (traceback attribute) - single: tb_lasti (traceback attribute) - pair: statement; try - - Special read-only attributes: - :attr:`tb_frame` points to the execution frame of the current level; - :attr:`tb_lineno` gives the line number where the exception occurred; - :attr:`tb_lasti` indicates the precise instruction. - The line number and last instruction in the traceback may differ from the - line number of its frame object if the exception occurred in a - :keyword:`try` statement with no matching except clause or with a - finally clause. - - Accessing ``tb_frame`` raises an :ref:`auditing event ` - ``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. - - .. index:: - single: tb_next (traceback attribute) - - Special writable attribute: :attr:`tb_next` is the next level in the stack - trace (towards the frame where the exception occurred), or ``None`` if - there is no next level. - - .. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. - - Slice objects - .. index:: pair: built-in function; slice - - Slice objects are used to represent slices for - :meth:`~object.__getitem__` - methods. They are also created by the built-in :func:`slice` function. - - .. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - - Special read-only attributes: :attr:`~slice.start` is the lower bound; - :attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step - value; each is ``None`` if omitted. These attributes can have any type. - - Slice objects support one method: - - .. method:: slice.indices(self, length) - - This method takes a single integer argument *length* and computes - information about the slice that the slice object would describe if - applied to a sequence of *length* items. It returns a tuple of three - integers; respectively these are the *start* and *stop* indices and the - *step* or stride length of the slice. Missing or out-of-bounds indices - are handled in a manner consistent with regular slices. - - Static method objects - Static method objects provide a way of defeating the transformation of function - objects to method objects described above. A static method object is a wrapper - around any other object, usually a user-defined method object. When a static - method object is retrieved from a class or a class instance, the object actually - returned is the wrapped object, which is not subject to any further - transformation. Static method objects are also callable. Static method - objects are created by the built-in :func:`staticmethod` constructor. - - Class method objects - A class method object, like a static method object, is a wrapper around another - object that alters the way in which that object is retrieved from classes and - class instances. The behaviour of class method objects upon such retrieval is - described above, under "User-defined methods". Class method objects are created - by the built-in :func:`classmethod` constructor. +-------------- + +.. index:: + single: internal type + single: types, internal + +A few types used internally by the interpreter are exposed to the user. Their +definitions may change with future versions of the interpreter, but they are +mentioned here for completeness. + +.. index:: bytecode, object; code, code object + + +Code objects +^^^^^^^^^^^^ + +Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. +The difference between a code object and a function object is that the function +object contains an explicit reference to the function's globals (the module in +which it was defined), while a code object contains no context; also the default +argument values are stored in the function object, not in the code object +(because they represent values calculated at run-time). Unlike function +objects, code objects are immutable and contain no references (directly or +indirectly) to mutable objects. + +.. index:: + single: co_argcount (code object attribute) + single: co_posonlyargcount (code object attribute) + single: co_kwonlyargcount (code object attribute) + single: co_code (code object attribute) + single: co_consts (code object attribute) + single: co_filename (code object attribute) + single: co_firstlineno (code object attribute) + single: co_flags (code object attribute) + single: co_lnotab (code object attribute) + single: co_name (code object attribute) + single: co_names (code object attribute) + single: co_nlocals (code object attribute) + single: co_stacksize (code object attribute) + single: co_varnames (code object attribute) + single: co_cellvars (code object attribute) + single: co_freevars (code object attribute) + single: co_qualname (code object attribute) + +Special read-only attributes: :attr:`co_name` gives the function name; +:attr:`co_qualname` gives the fully qualified function name; +:attr:`co_argcount` is the total number of positional arguments +(including positional-only arguments and arguments with default values); +:attr:`co_posonlyargcount` is the number of positional-only arguments +(including arguments with default values); :attr:`co_kwonlyargcount` is +the number of keyword-only arguments (including arguments with default +values); :attr:`co_nlocals` is the number of local variables used by the +function (including arguments); :attr:`co_varnames` is a tuple containing +the names of the local variables (starting with the argument names); +:attr:`co_cellvars` is a tuple containing the names of local variables +that are referenced by nested functions; :attr:`co_freevars` is a tuple +containing the names of free variables; :attr:`co_code` is a string +representing the sequence of bytecode instructions; :attr:`co_consts` is +a tuple containing the literals used by the bytecode; :attr:`co_names` is +a tuple containing the names used by the bytecode; :attr:`co_filename` is +the filename from which the code was compiled; :attr:`co_firstlineno` is +the first line number of the function; :attr:`co_lnotab` is a string +encoding the mapping from bytecode offsets to line numbers (for details +see the source code of the interpreter, is deprecated since 3.12 +and may be removed in 3.14); :attr:`co_stacksize` is the +required stack size; :attr:`co_flags` is an integer encoding a number +of flags for the interpreter. + +.. index:: pair: object; generator + +The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if +the function uses the ``*arguments`` syntax to accept an arbitrary number of +positional arguments; bit ``0x08`` is set if the function uses the +``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set +if the function is a generator. + +Future feature declarations (``from __future__ import division``) also use bits +in :attr:`co_flags` to indicate whether a code object was compiled with a +particular feature enabled: bit ``0x2000`` is set if the function was compiled +with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier +versions of Python. + +Other bits in :attr:`co_flags` are reserved for internal use. + +.. index:: single: documentation string + +If a code object represents a function, the first item in :attr:`co_consts` is +the documentation string of the function, or ``None`` if undefined. + +.. method:: codeobject.co_positions() + + Returns an iterable over the source code positions of each bytecode + instruction in the code object. + + The iterator returns tuples containing the ``(start_line, end_line, + start_column, end_column)``. The *i-th* tuple corresponds to the + position of the source code that compiled to the *i-th* instruction. + Column information is 0-indexed utf-8 byte offsets on the given source + line. + + This positional information can be missing. A non-exhaustive lists of + cases where this may happen: + + - Running the interpreter with :option:`-X` ``no_debug_ranges``. + - Loading a pyc file compiled while using :option:`-X` ``no_debug_ranges``. + - Position tuples corresponding to artificial instructions. + - Line and column numbers that can't be represented due to + implementation specific limitations. + + When this occurs, some or all of the tuple elements can be + :const:`None`. + + .. versionadded:: 3.11 + + .. note:: + This feature requires storing column positions in code objects which may + result in a small increase of disk usage of compiled Python files or + interpreter memory usage. To avoid storing the extra information and/or + deactivate printing the extra traceback information, the + :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` + environment variable can be used. + + +.. _frame-objects: + +Frame objects +^^^^^^^^^^^^^ + +.. index:: pair: object; frame + +Frame objects represent execution frames. They may occur in traceback objects +(see below), and are also passed to registered trace functions. + +.. index:: + single: f_back (frame attribute) + single: f_code (frame attribute) + single: f_globals (frame attribute) + single: f_locals (frame attribute) + single: f_lasti (frame attribute) + single: f_builtins (frame attribute) + +Special read-only attributes: :attr:`f_back` is to the previous stack frame +(towards the caller), or ``None`` if this is the bottom stack frame; +:attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` +is the dictionary used to look up local variables; :attr:`f_globals` is used for +global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; +:attr:`f_lasti` gives the precise instruction (this is an index into the +bytecode string of the code object). + +Accessing ``f_code`` raises an :ref:`auditing event ` +``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + +.. index:: + single: f_trace (frame attribute) + single: f_trace_lines (frame attribute) + single: f_trace_opcodes (frame attribute) + single: f_lineno (frame attribute) + +Special writable attributes: :attr:`f_trace`, if not ``None``, is a function +called for various events during code execution (this is used by the debugger). +Normally an event is triggered for each new source line - this can be +disabled by setting :attr:`f_trace_lines` to :const:`False`. + +Implementations *may* allow per-opcode events to be requested by setting +:attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to +undefined interpreter behaviour if exceptions raised by the trace +function escape to the function being traced. + +:attr:`f_lineno` is the current line number of the frame --- writing to this +from within a trace function jumps to the given line (only for the bottom-most +frame). A debugger can implement a Jump command (aka Set Next Statement) +by writing to f_lineno. + +Frame objects support one method: + +.. method:: frame.clear() + + This method clears all references to local variables held by the + frame. Also, if the frame belonged to a generator, the generator + is finalized. This helps break reference cycles involving frame + objects (for example when catching an exception and storing its + traceback for later use). + + :exc:`RuntimeError` is raised if the frame is currently executing. + + .. versionadded:: 3.4 + + +.. _traceback-objects: + +Traceback objects +^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; traceback + pair: stack; trace + pair: exception; handler + pair: execution; stack + single: exc_info (in module sys) + single: last_traceback (in module sys) + single: sys.exc_info + single: sys.exception + single: sys.last_traceback + +Traceback objects represent a stack trace of an exception. A traceback object +is implicitly created when an exception occurs, and may also be explicitly +created by calling :class:`types.TracebackType`. + +For implicitly created tracebacks, when the search for an exception handler +unwinds the execution stack, at each unwound level a traceback object is +inserted in front of the current traceback. When an exception handler is +entered, the stack trace is made available to the program. (See section +:ref:`try`.) It is accessible as the third item of the +tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute +of the caught exception. + +When the program contains no suitable +handler, the stack trace is written (nicely formatted) to the standard error +stream; if the interpreter is interactive, it is also made available to the user +as ``sys.last_traceback``. + +For explicitly created tracebacks, it is up to the creator of the traceback +to determine how the ``tb_next`` attributes should be linked to form a +full stack trace. + +.. index:: + single: tb_frame (traceback attribute) + single: tb_lineno (traceback attribute) + single: tb_lasti (traceback attribute) + pair: statement; try + +Special read-only attributes: +:attr:`tb_frame` points to the execution frame of the current level; +:attr:`tb_lineno` gives the line number where the exception occurred; +:attr:`tb_lasti` indicates the precise instruction. +The line number and last instruction in the traceback may differ from the +line number of its frame object if the exception occurred in a +:keyword:`try` statement with no matching except clause or with a +finally clause. + +Accessing ``tb_frame`` raises an :ref:`auditing event ` +``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. + +.. index:: + single: tb_next (traceback attribute) + +Special writable attribute: :attr:`tb_next` is the next level in the stack +trace (towards the frame where the exception occurred), or ``None`` if +there is no next level. + +.. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code, + and the ``tb_next`` attribute of existing instances can be updated. + + +Slice objects +^^^^^^^^^^^^^ + +.. index:: pair: built-in function; slice + +Slice objects are used to represent slices for +:meth:`~object.__getitem__` +methods. They are also created by the built-in :func:`slice` function. + +.. index:: + single: start (slice object attribute) + single: stop (slice object attribute) + single: step (slice object attribute) + +Special read-only attributes: :attr:`~slice.start` is the lower bound; +:attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step +value; each is ``None`` if omitted. These attributes can have any type. + +Slice objects support one method: + +.. method:: slice.indices(self, length) + + This method takes a single integer argument *length* and computes + information about the slice that the slice object would describe if + applied to a sequence of *length* items. It returns a tuple of three + integers; respectively these are the *start* and *stop* indices and the + *step* or stride length of the slice. Missing or out-of-bounds indices + are handled in a manner consistent with regular slices. + + +Static method objects +^^^^^^^^^^^^^^^^^^^^^ + +Static method objects provide a way of defeating the transformation of function +objects to method objects described above. A static method object is a wrapper +around any other object, usually a user-defined method object. When a static +method object is retrieved from a class or a class instance, the object actually +returned is the wrapped object, which is not subject to any further +transformation. Static method objects are also callable. Static method +objects are created by the built-in :func:`staticmethod` constructor. + + +Class method objects +^^^^^^^^^^^^^^^^^^^^ + +A class method object, like a static method object, is a wrapper around another +object that alters the way in which that object is retrieved from classes and +class instances. The behaviour of class method objects upon such retrieval is +described above, under "User-defined methods". Class method objects are created +by the built-in :func:`classmethod` constructor. .. _specialnames: From cb1184280b3fb369a07abb4153aa36829cf1df9b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 25 Aug 2023 13:28:47 +0200 Subject: [PATCH 330/598] Docs: Resolve Sphinx warnings in dis.rst (#108476) - Link to the code objects reference - Suppress link to deliberately undocumented builtins.__build_class__ - Suppress links for example methods --- Doc/library/dis.rst | 20 +++++++++++--------- Doc/reference/datamodel.rst | 4 +++- Doc/tools/.nitignore | 1 - 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 7c3363b143ab58..6cd5f181c8433a 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -43,13 +43,13 @@ interpreter. adaptive bytecode can be shown by passing ``adaptive=True``. -Example: Given the function :func:`myfunc`:: +Example: Given the function :func:`!myfunc`:: def myfunc(alist): return len(alist) the following command can be used to display the disassembly of -:func:`myfunc`: +:func:`!myfunc`: .. doctest:: @@ -862,7 +862,7 @@ iterations of the loop. .. opcode:: LOAD_BUILD_CLASS - Pushes :func:`builtins.__build_class__` onto the stack. It is later called + Pushes :func:`!builtins.__build_class__` onto the stack. It is later called to construct a class. @@ -920,14 +920,14 @@ iterations of the loop. .. opcode:: STORE_NAME (namei) Implements ``name = STACK.pop()``. *namei* is the index of *name* in the attribute - :attr:`co_names` of the code object. The compiler tries to use - :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. + :attr:`!co_names` of the :ref:`code object `. + The compiler tries to use :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. .. opcode:: DELETE_NAME (namei) - Implements ``del name``, where *namei* is the index into :attr:`co_names` - attribute of the code object. + Implements ``del name``, where *namei* is the index into :attr:`!co_names` + attribute of the :ref:`code object `. .. opcode:: UNPACK_SEQUENCE (count) @@ -966,7 +966,8 @@ iterations of the loop. value = STACK.pop() obj.name = value - where *namei* is the index of name in :attr:`co_names`. + where *namei* is the index of name in :attr:`!co_names` of the + :ref:`code object `. .. opcode:: DELETE_ATTR (namei) @@ -975,7 +976,8 @@ iterations of the loop. obj = STACK.pop() del obj.name - where *namei* is the index of name into :attr:`co_names`. + where *namei* is the index of name into :attr:`!co_names` of the + :ref:`code object `. .. opcode:: STORE_GLOBAL (namei) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 832a9150c16ad9..cc81f73f824352 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1039,12 +1039,14 @@ A few types used internally by the interpreter are exposed to the user. Their definitions may change with future versions of the interpreter, but they are mentioned here for completeness. -.. index:: bytecode, object; code, code object +.. _code-objects: Code objects ^^^^^^^^^^^^ +.. index:: bytecode, object; code, code object + Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. The difference between a code object and a function object is that the function object contains an explicit reference to the function's globals (the module in diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index bb5515b5610951..953d10b19e0ccc 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -57,7 +57,6 @@ Doc/library/dbm.rst Doc/library/decimal.rst Doc/library/devmode.rst Doc/library/difflib.rst -Doc/library/dis.rst Doc/library/doctest.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst From bc2f9e625865bc13549100c0eb76d3b9c0990f1a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 25 Aug 2023 13:02:10 +0100 Subject: [PATCH 331/598] gh-107932: Fix merge conflict in test_dis (GH-108478) Fix merge conflict in test_dis --- Lib/test/test_dis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f51c7dae8a4ce7..dacd6f6da2c5a9 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1728,7 +1728,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=False, line_number=7, is_jump_target=False, positions=None), Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=False, line_number=7, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=False, line_number=7, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=True, line_number=8, is_jump_target=True, positions=None), Instruction(opname='JUMP_FORWARD', opcode=125, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=False, line_number=8, is_jump_target=False, positions=None), Instruction(opname='END_FOR', opcode=24, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=3, is_jump_target=True, positions=None), @@ -1760,7 +1760,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=True, line_number=11, is_jump_target=True, positions=None), Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=False, line_number=11, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=False, line_number=11, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=142, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, is_jump_target=False, positions=None), From 75903f29f6fed718d06412f5a35105a6952961ce Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 25 Aug 2023 14:31:52 +0200 Subject: [PATCH 332/598] gh-108364: Simplify quoting values and identifiers in sqlite3's iterdump() (#108472) --- Lib/sqlite3/dump.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 07b9da10b920f9..cf73cc33810cb1 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -7,6 +7,14 @@ # future enhancements, you should normally quote any identifier that # is an English language word, even if you do not have to." +def _quote_name(name): + return '"{0}"'.format(name.replace('"', '""')) + + +def _quote_value(value): + return "'{0}'".format(value.replace("'", "''")) + + def _iterdump(connection): """ Returns an iterator to the dump of the database in an SQL text format. @@ -31,11 +39,11 @@ def _iterdump(connection): sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': - rows = cu.execute('SELECT * FROM "sqlite_sequence";').fetchall() + rows = cu.execute('SELECT * FROM "sqlite_sequence";') sqlite_sequence = ['DELETE FROM "sqlite_sequence"'] sqlite_sequence += [ - f'INSERT INTO "sqlite_sequence" VALUES(\'{row[0]}\',{row[1]})' - for row in rows + f'INSERT INTO "sqlite_sequence" VALUES({_quote_value(table_name)},{seq_value})' + for table_name, seq_value in rows.fetchall() ] continue elif table_name == 'sqlite_stat1': @@ -53,12 +61,15 @@ def _iterdump(connection): yield('{0};'.format(sql)) # Build the insert statement for each row of the current table - table_name_ident = table_name.replace('"', '""') - res = cu.execute('PRAGMA table_info("{0}")'.format(table_name_ident)) + table_name_ident = _quote_name(table_name) + res = cu.execute(f'PRAGMA table_info({table_name_ident})') column_names = [str(table_info[1]) for table_info in res.fetchall()] - q = """SELECT 'INSERT INTO "{0}" VALUES({1})' FROM "{0}";""".format( + q = "SELECT 'INSERT INTO {0} VALUES('{1}')' FROM {0};".format( table_name_ident, - ",".join("""'||quote("{0}")||'""".format(col.replace('"', '""')) for col in column_names)) + "','".join( + "||quote({0})||".format(_quote_name(col)) for col in column_names + ) + ) query_res = cu.execute(q) for row in query_res: yield("{0};".format(row[0])) From 66b4d9c9f0b8a935b5d464abd2f6ee0253832fd9 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 26 Aug 2023 00:23:39 +0900 Subject: [PATCH 333/598] gh-107265: Revert "Ensure _PyCode_Quicken does not handle ENTER_EXECUTOR" (#108485) This reverts commit d6ac5c7b105fe57266bd71248e3ada41fedb5ba9. Reason: the assert we just added could be triggered (see issue). --- Python/specialize.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index fac3c3ea57d004..a467f163f2ca9d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -300,8 +300,6 @@ _PyCode_Quicken(PyCodeObject *code) for (int i = 0; i < Py_SIZE(code); i++) { opcode = _Py_GetBaseOpcode(code, i); assert(opcode < MIN_INSTRUMENTED_OPCODE); - // _PyCode_Quicken is only called when initializing a fresh code object. - assert(opcode != ENTER_EXECUTOR); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); From 53470184091f6fe1c7a1cf4de8fd90dc2ced7654 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 25 Aug 2023 09:40:27 -0700 Subject: [PATCH 334/598] gh-108311: Fix test_store_attr_with_hint by disabling optimizer in decorator (#108312) See https://github.com/python/cpython/issues/108311#issuecomment-1693569380 --------- Co-authored-by: AlexWaygood --- Lib/test/test_opcache.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 1baa10cbdfdef2..692e03fbb5e084 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -8,6 +8,19 @@ import _testinternalcapi +def disabling_optimizer(func): + def wrapper(*args, **kwargs): + import _testinternalcapi + old_opt = _testinternalcapi.get_optimizer() + _testinternalcapi.set_optimizer(None) + try: + return func(*args, **kwargs) + finally: + _testinternalcapi.set_optimizer(old_opt) + + return wrapper + + class TestLoadSuperAttrCache(unittest.TestCase): def test_descriptor_not_double_executed_on_spec_fail(self): calls = [] @@ -502,6 +515,7 @@ def assert_specialized(self, f, opname): opnames = {instruction.opname for instruction in instructions} self.assertIn(opname, opnames) + @disabling_optimizer def assert_races_do_not_crash( self, opname, get_items, read, write, *, check_items=False ): From 6895ddf6cb2bada7e392eb971c88ded03d8fc79e Mon Sep 17 00:00:00 2001 From: Philipp A Date: Fri, 25 Aug 2023 18:53:11 +0200 Subject: [PATCH 335/598] =?UTF-8?q?gh-102211:=20Document=20`re.{Pattern,Ma?= =?UTF-8?q?tch}`=E2=80=99s=20existence=20(#102212)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jelle Zijlstra Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Alex Waygood --- Doc/library/re.rst | 67 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index ab201e2483f8e3..e506b346ec379d 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -856,18 +856,17 @@ Functions .. function:: search(pattern, string, flags=0) Scan through *string* looking for the first location where the regular expression - *pattern* produces a match, and return a corresponding :ref:`match object - `. Return ``None`` if no position in the string matches the - pattern; note that this is different from finding a zero-length match at some - point in the string. + *pattern* produces a match, and return a corresponding :class:`~re.Match`. Return + ``None`` if no position in the string matches the pattern; note that this is + different from finding a zero-length match at some point in the string. .. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular - expression *pattern*, return a corresponding :ref:`match object - `. Return ``None`` if the string does not match the pattern; - note that this is different from a zero-length match. + expression *pattern*, return a corresponding :class:`~re.Match`. Return + ``None`` if the string does not match the pattern; note that this is + different from a zero-length match. Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match at the beginning of the string and not at the beginning of each line. @@ -879,9 +878,8 @@ Functions .. function:: fullmatch(pattern, string, flags=0) If the whole *string* matches the regular expression *pattern*, return a - corresponding :ref:`match object `. Return ``None`` if the - string does not match the pattern; note that this is different from a - zero-length match. + corresponding :class:`~re.Match`. Return ``None`` if the string does not match + the pattern; note that this is different from a zero-length match. .. versionadded:: 3.4 @@ -959,7 +957,7 @@ Functions .. function:: finditer(pattern, string, flags=0) - Return an :term:`iterator` yielding :ref:`match objects ` over + Return an :term:`iterator` yielding :class:`~re.Match` objects over all non-overlapping matches for the RE *pattern* in *string*. The *string* is scanned left-to-right, and matches are returned in the order found. Empty matches are included in the result. @@ -987,8 +985,8 @@ Functions 'static PyObject*\npy_myfunc(void)\n{' If *repl* is a function, it is called for every non-overlapping occurrence of - *pattern*. The function takes a single :ref:`match object ` - argument, and returns the replacement string. For example:: + *pattern*. The function takes a single :class:`~re.Match` argument, and returns + the replacement string. For example:: >>> def dashrepl(matchobj): ... if matchobj.group(0) == '-': return ' ' @@ -999,7 +997,7 @@ Functions >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) 'Baked Beans & Spam' - The pattern may be a string or a :ref:`pattern object `. + The pattern may be a string or a :class:`~re.Pattern`. The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all @@ -1131,16 +1129,20 @@ Exceptions Regular Expression Objects -------------------------- -Compiled regular expression objects support the following methods and -attributes: +.. class:: Pattern + + Compiled regular expression object returned by :func:`re.compile`. + + .. versionchanged:: 3.9 + :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. + See :ref:`types-genericalias`. .. method:: Pattern.search(string[, pos[, endpos]]) Scan through *string* looking for the first location where this regular - expression produces a match, and return a corresponding :ref:`match object - `. Return ``None`` if no position in the string matches the - pattern; note that this is different from finding a zero-length match at some - point in the string. + expression produces a match, and return a corresponding :class:`~re.Match`. + Return ``None`` if no position in the string matches the pattern; note that + this is different from finding a zero-length match at some point in the string. The optional second parameter *pos* gives an index in the string where the search is to start; it defaults to ``0``. This is not completely equivalent to @@ -1164,9 +1166,9 @@ attributes: .. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular - expression, return a corresponding :ref:`match object `. - Return ``None`` if the string does not match the pattern; note that this is - different from a zero-length match. + expression, return a corresponding :class:`~re.Match`. Return ``None`` if the + string does not match the pattern; note that this is different from a + zero-length match. The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: @@ -1183,8 +1185,8 @@ attributes: .. method:: Pattern.fullmatch(string[, pos[, endpos]]) If the whole *string* matches this regular expression, return a corresponding - :ref:`match object `. Return ``None`` if the string does not - match the pattern; note that this is different from a zero-length match. + :class:`~re.Match`. Return ``None`` if the string does not match the pattern; + note that this is different from a zero-length match. The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: @@ -1270,8 +1272,13 @@ when there is no match, you can test whether there was a match with a simple if match: process(match) -Match objects support the following methods and attributes: +.. class:: Match + + Match object returned by successful ``match``\ es and ``search``\ es. + .. versionchanged:: 3.9 + :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. + See :ref:`types-genericalias`. .. method:: Match.expand(template) @@ -1715,10 +1722,10 @@ Finding all Adverbs and their Positions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If one wants more information about all matches of a pattern than the matched -text, :func:`finditer` is useful as it provides :ref:`match objects -` instead of strings. Continuing with the previous example, if -a writer wanted to find all of the adverbs *and their positions* in -some text, they would use :func:`finditer` in the following manner:: +text, :func:`finditer` is useful as it provides :class:`~re.Match` objects +instead of strings. Continuing with the previous example, if a writer wanted +to find all of the adverbs *and their positions* in some text, they would use +:func:`finditer` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." >>> for m in re.finditer(r"\w+ly\b", text): From 5a25daa5124606e3973265c6f6f9757a3c366cdc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 25 Aug 2023 18:08:29 +0100 Subject: [PATCH 336/598] gh-104504: Cases generator: enable mypy's `possibly-undefined` error code (#108454) --- Tools/cases_generator/generate_cases.py | 13 +++++++++---- Tools/cases_generator/mypy.ini | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 9400a0bce462c8..f89572fc496a70 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -156,6 +156,8 @@ def effect_str(effects: list[StackEffect]) -> str: return str(n_effect) instr: AnyInstruction | None + popped: str | None = None + pushed: str | None = None match thing: case parsing.InstDef(): if thing.kind != "op" or self.instrs[thing.name].is_viable_uop(): @@ -173,7 +175,7 @@ def effect_str(effects: list[StackEffect]) -> str: instr = self.pseudo_instrs[thing.name] # Calculate stack effect, and check that it's the the same # for all targets. - for idx, target in enumerate(self.pseudos[thing.name].targets): + for target in self.pseudos[thing.name].targets: target_instr = self.instrs.get(target) # Currently target is always an instr. This could change # in the future, e.g., if we have a pseudo targetting a @@ -181,13 +183,14 @@ def effect_str(effects: list[StackEffect]) -> str: assert target_instr target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) - if idx == 0: + if popped is None: popped, pushed = target_popped, target_pushed else: assert popped == target_popped assert pushed == target_pushed case _: assert_never(thing) + assert popped is not None and pushed is not None return instr, popped, pushed @contextlib.contextmanager @@ -376,6 +379,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Compute the set of all instruction formats. all_formats: set[str] = set() for thing in self.everything: + format: str | None = None match thing: case OverriddenInstructionPlaceHolder(): continue @@ -384,15 +388,16 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Macro(): format = self.macro_instrs[thing.name].instr_fmt case parsing.Pseudo(): - for idx, target in enumerate(self.pseudos[thing.name].targets): + for target in self.pseudos[thing.name].targets: target_instr = self.instrs.get(target) assert target_instr - if idx == 0: + if format is None: format = target_instr.instr_fmt else: assert format == target_instr.instr_fmt case _: assert_never(thing) + assert format is not None all_formats.add(format) # Turn it into a sorted list of enum values. diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index 54ccb8c5c1a3a2..e7175e263350b2 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -9,5 +9,5 @@ python_version = 3.10 # ...And be strict: strict = True strict_concatenate = True -enable_error_code = ignore-without-code,redundant-expr,truthy-bool +enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True From 5f41376e93cdeb1a4714b0f08b5921b69e345dd6 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:33:59 +0100 Subject: [PATCH 337/598] gh-107901: add the HAS_EVAL_BREAK instruction flag (#108375) --- Include/internal/pycore_opcode_metadata.h | 36 ++++++++++++----------- Tools/cases_generator/analysis.py | 3 +- Tools/cases_generator/flags.py | 10 +++++-- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bf7e9dcbffad23..49089721c6ac20 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1158,12 +1158,14 @@ enum InstructionFormat { #define HAS_JUMP_FLAG (8) #define HAS_FREE_FLAG (16) #define HAS_LOCAL_FLAG (32) +#define HAS_EVAL_BREAK_FLAG (64) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) #define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG)) #define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) +#define OPCODE_HAS_EVAL_BREAK(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EVAL_BREAK_FLAG)) struct opcode_metadata { bool valid_entry; @@ -1196,8 +1198,8 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SI #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1333,10 +1335,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG }, [JUMP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1370,28 +1372,28 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, [KW_NAMES] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, + [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, 0 }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [CALL_NO_KW_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [MAKE_FUNCTION] = { true, INSTR_FMT_IX, 0 }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, 0 }, @@ -1404,7 +1406,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 72fb2d761dbfae..b3c3df1378b943 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -381,7 +381,8 @@ def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: # Make sure the targets have the same fmt fmts = list(set([t.instr_fmt for t in targets])) assert len(fmts) == 1 - assert len(list(set([t.instr_flags.bitmap() for t in targets]))) == 1 + ignored_flags = {'HAS_EVAL_BREAK_FLAG'} + assert len({t.instr_flags.bitmap(ignore=ignored_flags) for t in targets}) == 1 return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) def analyze_instruction( diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 536f093c7d6ede..193e7ff0d9ccfb 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -3,6 +3,7 @@ from formatting import Formatter import lexer as lx import parsing +from typing import AbstractSet @dataclasses.dataclass @@ -15,6 +16,7 @@ class InstructionFlags: HAS_JUMP_FLAG: bool HAS_FREE_FLAG: bool HAS_LOCAL_FLAG: bool + HAS_EVAL_BREAK_FLAG: bool def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @@ -37,11 +39,12 @@ def fromInstruction(instr: parsing.Node) -> "InstructionFlags": variable_used(instr, "GETLOCAL") or variable_used(instr, "SETLOCAL") ) and not has_free, + HAS_EVAL_BREAK_FLAG=variable_used(instr, "CHECK_EVAL_BREAKER"), ) @staticmethod def newEmpty() -> "InstructionFlags": - return InstructionFlags(False, False, False, False, False, False) + return InstructionFlags(False, False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: for name, value in dataclasses.asdict(other).items(): @@ -53,10 +56,11 @@ def names(self, value: bool | None = None) -> list[str]: return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] - def bitmap(self) -> int: + def bitmap(self, ignore: AbstractSet[str] = frozenset()) -> int: flags = 0 + assert all(hasattr(self, name) for name in ignore) for name in self.names(): - if getattr(self, name): + if getattr(self, name) and name not in ignore: flags |= self.bitmask[name] return flags From 4eae1e53425d3a816a26760f28d128a4f05c1da4 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 25 Aug 2023 12:12:59 -0700 Subject: [PATCH 338/598] GH-106581: Fix instrumentation in tier 2 (GH-108493) --- .../2023-08-10-00-00-48.gh-issue-106581.o7zDty.rst | 2 ++ Python/bytecodes.c | 10 ++++++---- Python/executor_cases.c.h | 10 ++++++---- Python/generated_cases.c.h | 10 ++++++---- 4 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-10-00-00-48.gh-issue-106581.o7zDty.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-10-00-00-48.gh-issue-106581.o7zDty.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-00-00-48.gh-issue-106581.o7zDty.rst new file mode 100644 index 00000000000000..dff1ebd9cf70f0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-00-00-48.gh-issue-106581.o7zDty.rst @@ -0,0 +1,2 @@ +Fix possible assertion failures and missing instrumentation events when +:envvar:`PYTHONUOPS` or :option:`-X uops <-X>` is enabled. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a5cb117c7631c6..a55460afea751c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -133,17 +133,19 @@ dummy_func( } inst(RESUME, (--)) { - #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); + #if TIER_ONE next_instr--; + #endif + #if TIER_TWO + goto deoptimize; + #endif } - else - #endif - if (oparg < 2) { + else if (oparg < 2) { CHECK_EVAL_BREAKER(); } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 85c60c276e0292..1283cc7ebbf9c4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8,17 +8,19 @@ } case RESUME: { - #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; + #if TIER_ONE next_instr--; + #endif + #if TIER_TWO + goto deoptimize; + #endif } - else - #endif - if (oparg < 2) { + else if (oparg < 2) { CHECK_EVAL_BREAKER(); } break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4aa16f8311a2a0..3f46f1a10a247f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,17 +8,19 @@ } TARGET(RESUME) { - #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; + #if TIER_ONE next_instr--; + #endif + #if TIER_TWO + goto deoptimize; + #endif } - else - #endif - if (oparg < 2) { + else if (oparg < 2) { CHECK_EVAL_BREAKER(); } DISPATCH(); From 1dd951097728d735d46a602fc43285d35b7b32cb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Aug 2023 23:22:08 +0200 Subject: [PATCH 339/598] gh-108494: Argument Clinic partial supports of Limited C API (#108495) Argument Clinic now has a partial support of the Limited API: * Add --limited option to clinic.c. * Add '_testclinic_limited' extension which is built with the limited C API version 3.13. * For now, hardcode in clinic.py that "_testclinic_limited.c" targets the limited C API. --- Lib/test/test_clinic.py | 40 ++++++++++- ...-08-25-22-40-12.gh-issue-108494.4RbDdu.rst | 2 + Modules/Setup.stdlib.in | 1 + Modules/_testclinic_limited.c | 69 +++++++++++++++++++ Modules/clinic/_testclinic_limited.c.h | 53 ++++++++++++++ Tools/build/generate_stdlib_module_names.py | 1 + Tools/clinic/clinic.py | 55 ++++++++++++--- 7 files changed, 208 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst create mode 100644 Modules/_testclinic_limited.c create mode 100644 Modules/clinic/_testclinic_limited.c.h diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f61a10b89a777a..9ee8f9ce835bbd 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -13,6 +13,7 @@ import os.path import re import sys +import types import unittest test_tools.skip_if_missing('clinic') @@ -21,6 +22,13 @@ from clinic import DSLParser +def default_namespace(): + ns = types.SimpleNamespace() + ns.force = False + ns.limited_capi = clinic.DEFAULT_LIMITED_CAPI + return ns + + def _make_clinic(*, filename='clinic_tests'): clang = clinic.CLanguage(None) c = clinic.Clinic(clang, filename=filename) @@ -52,6 +60,11 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, return cm.exception +class MockClinic: + def __init__(self): + self.limited_capi = clinic.DEFAULT_LIMITED_CAPI + + class ClinicWholeFileTest(TestCase): maxDiff = None @@ -691,8 +704,9 @@ def expect_parsing_failure( self, *, filename, expected_error, verify=True, output=None ): errmsg = re.escape(dedent(expected_error).strip()) + ns = default_namespace() with self.assertRaisesRegex(clinic.ClinicError, errmsg): - clinic.parse_file(filename) + clinic.parse_file(filename, ns=ns) def test_parse_file_no_extension(self) -> None: self.expect_parsing_failure( @@ -832,8 +846,9 @@ def _test(self, input, output): blocks = list(clinic.BlockParser(input, language)) writer = clinic.BlockPrinter(language) + mock_clinic = MockClinic() for block in blocks: - writer.print_block(block) + writer.print_block(block, clinic=mock_clinic) output = writer.f.getvalue() assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) @@ -3508,6 +3523,27 @@ def test_depr_multi(self): self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") +try: + import _testclinic_limited +except ImportError: + _testclinic_limited = None + +@unittest.skipIf(_testclinic_limited is None, "_testclinic_limited is missing") +class LimitedCAPIFunctionalTest(unittest.TestCase): + locals().update((name, getattr(_testclinic_limited, name)) + for name in dir(_testclinic_limited) if name.startswith('test_')) + + def test_my_int_func(self): + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func() + self.assertEqual(_testclinic_limited.my_int_func(3), 3) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func(1.0) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func("xyz") + + + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst new file mode 100644 index 00000000000000..2d611527b26092 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst @@ -0,0 +1,2 @@ +:ref:`Argument Clinic ` now has a partial support of the +:ref:`Limited API `. Patch by Victor Stinner. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 689f1d42ef0eee..6ed84953dd4216 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -161,6 +161,7 @@ @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c +@MODULE__TESTCLINIC_TRUE@_testclinic_limited _testclinic_limited.c # Some testing modules MUST be built as shared libraries. *shared* diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c new file mode 100644 index 00000000000000..6dd2745a7117c2 --- /dev/null +++ b/Modules/_testclinic_limited.c @@ -0,0 +1,69 @@ +// For now, only limited C API 3.13 is supported +#define Py_LIMITED_API 0x030d0000 + +/* Always enable assertions */ +#undef NDEBUG + +#include "Python.h" + + +#include "clinic/_testclinic_limited.c.h" + + +/*[clinic input] +module _testclinic_limited +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dd408149a4fc0dbb]*/ + + +/*[clinic input] +test_empty_function + +[clinic start generated code]*/ + +static PyObject * +test_empty_function_impl(PyObject *module) +/*[clinic end generated code: output=0f8aeb3ddced55cb input=0dd7048651ad4ae4]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +my_int_func -> int + + arg: int + / + +[clinic start generated code]*/ + +static int +my_int_func_impl(PyObject *module, int arg) +/*[clinic end generated code: output=761cd54582f10e4f input=16eb8bba71d82740]*/ +{ + return arg; +} + + +static PyMethodDef tester_methods[] = { + TEST_EMPTY_FUNCTION_METHODDEF + MY_INT_FUNC_METHODDEF + {NULL, NULL} +}; + +static struct PyModuleDef _testclinic_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testclinic_limited", + .m_size = 0, + .m_methods = tester_methods, +}; + +PyMODINIT_FUNC +PyInit__testclinic_limited(void) +{ + PyObject *m = PyModule_Create(&_testclinic_module); + if (m == NULL) { + return NULL; + } + return m; +} diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h new file mode 100644 index 00000000000000..730e967ad610da --- /dev/null +++ b/Modules/clinic/_testclinic_limited.c.h @@ -0,0 +1,53 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(test_empty_function__doc__, +"test_empty_function($module, /)\n" +"--\n" +"\n"); + +#define TEST_EMPTY_FUNCTION_METHODDEF \ + {"test_empty_function", (PyCFunction)test_empty_function, METH_NOARGS, test_empty_function__doc__}, + +static PyObject * +test_empty_function_impl(PyObject *module); + +static PyObject * +test_empty_function(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_empty_function_impl(module); +} + +PyDoc_STRVAR(my_int_func__doc__, +"my_int_func($module, arg, /)\n" +"--\n" +"\n"); + +#define MY_INT_FUNC_METHODDEF \ + {"my_int_func", (PyCFunction)my_int_func, METH_O, my_int_func__doc__}, + +static int +my_int_func_impl(PyObject *module, int arg); + +static PyObject * +my_int_func(PyObject *module, PyObject *arg_) +{ + PyObject *return_value = NULL; + int arg; + int _return_value; + + arg = PyLong_AsInt(arg_); + if (arg == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_int_func_impl(module, arg); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=07e2e8ed6923cd16 input=a9049054013a1b77]*/ diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 72f6923c7c316a..766a85d3d6f39e 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -28,6 +28,7 @@ '_testbuffer', '_testcapi', '_testclinic', + '_testclinic_limited', '_testconsole', '_testimportmultiple', '_testinternalcapi', diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a541c9f7e0760c..898347fd7120a2 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -63,6 +63,7 @@ version = '1' +DEFAULT_LIMITED_CAPI = False NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" CLINIC_PREFIXED_ARGS = { @@ -1360,7 +1361,21 @@ def parser_body( vararg ) nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" - if not new_or_init: + + if clinic.limited_capi: + # positional-or-keyword arguments + flags = "METH_VARARGS|METH_KEYWORDS" + + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + parser_code = [normalize_snippet(""" + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, + {parse_arguments})) + goto exit; + """, indent=4)] + argname_fmt = 'args[%d]' + declarations = "" + + elif not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS argname_fmt = 'args[%d]' @@ -2111,7 +2126,8 @@ def print_block( self, block: Block, *, - core_includes: bool = False + core_includes: bool = False, + clinic: Clinic | None = None, ) -> None: input = block.input output = block.output @@ -2140,7 +2156,11 @@ def print_block( write("\n") output = '' - if core_includes: + if clinic: + limited_capi = clinic.limited_capi + else: + limited_capi = DEFAULT_LIMITED_CAPI + if core_includes and not limited_capi: output += textwrap.dedent(""" #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head @@ -2344,6 +2364,7 @@ def __init__( *, filename: str, verify: bool = True, + limited_capi: bool = False, ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) @@ -2353,6 +2374,7 @@ def __init__( fail("Custom printers are broken right now") self.printer = printer or BlockPrinter(language) self.verify = verify + self.limited_capi = limited_capi self.filename = filename self.modules: ModuleDict = {} self.classes: ClassDict = {} @@ -2450,7 +2472,7 @@ def parse(self, input: str) -> str: self.parsers[dsl_name] = parsers[dsl_name](self) parser = self.parsers[dsl_name] parser.parse(block) - printer.print_block(block) + printer.print_block(block, clinic=self) # these are destinations not buffers for name, destination in self.destinations.items(): @@ -2465,7 +2487,7 @@ def parse(self, input: str) -> str: block.input = "dump " + name + "\n" warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") printer.write("\n") - printer.print_block(block) + printer.print_block(block, clinic=self) continue if destination.type == 'file': @@ -2490,7 +2512,7 @@ def parse(self, input: str) -> str: block.input = 'preserve\n' printer_2 = BlockPrinter(self.language) - printer_2.print_block(block, core_includes=True) + printer_2.print_block(block, core_includes=True, clinic=self) write_file(destination.filename, printer_2.f.getvalue()) continue @@ -2536,9 +2558,15 @@ def __repr__(self) -> str: def parse_file( filename: str, *, - verify: bool = True, - output: str | None = None + ns: argparse.Namespace, + output: str | None = None, ) -> None: + verify = not ns.force + limited_capi = ns.limited_capi + # XXX Temporary solution + if os.path.basename(filename) == '_testclinic_limited.c': + print(f"{filename} uses limited C API") + limited_capi = True if not output: output = filename @@ -2560,7 +2588,10 @@ def parse_file( return assert isinstance(language, CLanguage) - clinic = Clinic(language, verify=verify, filename=filename) + clinic = Clinic(language, + verify=verify, + filename=filename, + limited_capi=limited_capi) cooked = clinic.parse(raw) write_file(output, cooked) @@ -5987,6 +6018,8 @@ def create_cli() -> argparse.ArgumentParser: cmdline.add_argument("--exclude", type=str, action="append", help=("a file to exclude in --make mode; " "can be given multiple times")) + cmdline.add_argument("--limited", dest="limited_capi", action='store_true', + help="use the Limited C API") cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", help="the list of files to process") return cmdline @@ -6077,7 +6110,7 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: continue if ns.verbose: print(path) - parse_file(path, verify=not ns.force) + parse_file(path, ns=ns) return if not ns.filename: @@ -6089,7 +6122,7 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: for filename in ns.filename: if ns.verbose: print(filename) - parse_file(filename, output=ns.output, verify=not ns.force) + parse_file(filename, output=ns.output, ns=ns) def main(argv: list[str] | None = None) -> NoReturn: From 73d33c1a3078c5f2588c89d61e1a17a1b2a26c34 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Aug 2023 00:01:10 +0200 Subject: [PATCH 340/598] gh-107603: Argument Clinic can emit includes (#108486) * Add Clinic.add_include() method * Add CConverter.include and CConverter.add_include() * Printer.print_block() gets a second parameter: clinic. * Remove duplicated declaration of "clinic" global variable. --- Tools/clinic/clinic.py | 58 +++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 898347fd7120a2..3d3ab4bf81a5b5 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1075,6 +1075,11 @@ def output_templates( del parameters[0] converters = [p.converter for p in parameters] + # Copy includes from parameters to Clinic + for converter in converters: + if converter.include: + clinic.add_include(*converter.include) + has_option_groups = parameters and (parameters[0].group or parameters[-1].group) default_return_converter = f.return_converter.type == 'PyObject *' new_or_init = f.kind.new_or_init @@ -2126,8 +2131,8 @@ def print_block( self, block: Block, *, + clinic: Clinic, core_includes: bool = False, - clinic: Clinic | None = None, ) -> None: input = block.input output = block.output @@ -2156,18 +2161,22 @@ def print_block( write("\n") output = '' - if clinic: - limited_capi = clinic.limited_capi - else: - limited_capi = DEFAULT_LIMITED_CAPI - if core_includes and not limited_capi: - output += textwrap.dedent(""" - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # include "pycore_gc.h" // PyGC_Head - # include "pycore_runtime.h" // _Py_ID() - #endif - - """) + if core_includes: + if not clinic.limited_capi: + output += textwrap.dedent(""" + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # include "pycore_gc.h" // PyGC_Head + # include "pycore_runtime.h" // _Py_ID() + #endif + + """) + + if clinic is not None: + # Emit optional includes + for include, reason in sorted(clinic.includes.items()): + line = f'#include "{include}"' + line = line.ljust(35) + f'// {reason}\n' + output += line input = ''.join(block.input) output += ''.join(block.output) @@ -2311,7 +2320,7 @@ def __init__(self, clinic: Clinic) -> None: ... def parse(self, block: Block) -> None: ... -clinic = None +clinic: Clinic | None = None class Clinic: presets_text = """ @@ -2379,6 +2388,9 @@ def __init__( self.modules: ModuleDict = {} self.classes: ClassDict = {} self.functions: list[Function] = [] + # dict: include name => reason + # Example: 'pycore_long.h' => '_PyLong_UnsignedShort_Converter()' + self.includes: dict[str, str] = {} self.line_prefix = self.line_suffix = '' @@ -2437,6 +2449,12 @@ def __init__( global clinic clinic = self + def add_include(self, name: str, reason: str) -> None: + if name in self.includes: + # Mention a single reason is enough, no need to list all of them + return + self.includes[name] = reason + def add_destination( self, name: str, @@ -3066,6 +3084,10 @@ class CConverter(metaclass=CConverterAutoRegister): # Only set by self_converter. signature_name: str | None = None + # Optional (name, reason) include which generate a line like: + # "#include "name" // reason" + include: tuple[str, str] | None = None + # keep in sync with self_converter.__init__! def __init__(self, # Positional args: @@ -3370,6 +3392,11 @@ def parser_name(self) -> str: else: return self.name + def add_include(self, name: str, reason: str) -> None: + if self.include is not None: + raise ValueError("a converter only supports a single include") + self.include = (name, reason) + type_checks = { '&PyLong_Type': ('PyLong_Check', 'int'), '&PyTuple_Type': ('PyTuple_Check', 'tuple'), @@ -5989,9 +6016,6 @@ def do_post_block_processing_cleanup(self, lineno: int) -> None: } -clinic = None - - def create_cli() -> argparse.ArgumentParser: cmdline = argparse.ArgumentParser( prog="clinic.py", From 86bc9e35c4aaf3bcc045ddd998844ffb64fec3a2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Aug 2023 00:39:24 +0200 Subject: [PATCH 341/598] gh-108494: AC supports pos-only args in limited C API (#108498) AC now checks for "#define Py_LIMITED_API" pattern to use the limited C API. --- Lib/test/test_clinic.py | 11 +++++++++ Modules/_testclinic_limited.c | 18 ++++++++++++++ Modules/clinic/_testclinic_limited.c.h | 34 +++++++++++++++++++++++++- Tools/clinic/clinic.py | 24 +++++++++++++++--- 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 9ee8f9ce835bbd..e7039d59070597 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3542,6 +3542,17 @@ def test_my_int_func(self): with self.assertRaises(TypeError): _testclinic_limited.my_int_func("xyz") + def test_my_int_sum(self): + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum() + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1) + self.assertEqual(_testclinic_limited.my_int_sum(1, 2), 3) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1.0, 2) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1, "str") + class PermutationTests(unittest.TestCase): diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 6dd2745a7117c2..0b606c9857fc40 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -45,9 +45,27 @@ my_int_func_impl(PyObject *module, int arg) } +/*[clinic input] +my_int_sum -> int + + x: int + y: int + / + +[clinic start generated code]*/ + +static int +my_int_sum_impl(PyObject *module, int x, int y) +/*[clinic end generated code: output=3e52db9ab5f37e2f input=0edb6796813bf2d3]*/ +{ + return x + y; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF + MY_INT_SUM_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index 730e967ad610da..9b0032528ff746 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -50,4 +50,36 @@ my_int_func(PyObject *module, PyObject *arg_) exit: return return_value; } -/*[clinic end generated code: output=07e2e8ed6923cd16 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(my_int_sum__doc__, +"my_int_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_INT_SUM_METHODDEF \ + {"my_int_sum", (PyCFunction)my_int_sum, METH_VARARGS, my_int_sum__doc__}, + +static int +my_int_sum_impl(PyObject *module, int x, int y); + +static PyObject * +my_int_sum(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int x; + int y; + int _return_value; + + if (!PyArg_ParseTuple(args, "ii:my_int_sum", + &x, &y)) + goto exit; + _return_value = my_int_sum_impl(module, x, y); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=f9f7209255bb969e input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3d3ab4bf81a5b5..a6974bd81e2d07 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -78,6 +78,7 @@ "noptargs", "return_value", } +LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') class Sentinels(enum.Enum): @@ -1249,6 +1250,22 @@ def parser_body( parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') + elif not requires_defining_class and pos_only == len(parameters) - pseudo_args and clinic.limited_capi: + # positional-only for the limited C API + flags = "METH_VARARGS" + + parser_prototype = self.PARSER_PROTOTYPE_VARARGS + parser_code = [normalize_snippet(""" + if (!PyArg_ParseTuple(args, "{format_units}:{name}", + {parse_arguments})) + goto exit; + """, indent=4)] + argname_fmt = 'args[%d]' + declarations = "" + + parser_definition = parser_body(parser_prototype, *parser_code, + declarations=declarations) + elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: if not new_or_init: # positional-only, but no option groups @@ -2581,10 +2598,6 @@ def parse_file( ) -> None: verify = not ns.force limited_capi = ns.limited_capi - # XXX Temporary solution - if os.path.basename(filename) == '_testclinic_limited.c': - print(f"{filename} uses limited C API") - limited_capi = True if not output: output = filename @@ -2605,6 +2618,9 @@ def parse_file( if not find_start_re.search(raw): return + if LIMITED_CAPI_REGEX.search(raw): + limited_capi = True + assert isinstance(language, CLanguage) clinic = Clinic(language, verify=verify, From 713afb8804666405f29115cf459b591308e3ab54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Aug 2023 02:24:27 +0200 Subject: [PATCH 342/598] gh-106320: Remove private _PyLong converter functions (#108499) Move these private functions to the internal C API (pycore_long.h): * _PyLong_UnsignedInt_Converter() * _PyLong_UnsignedLongLong_Converter() * _PyLong_UnsignedLong_Converter() * _PyLong_UnsignedShort_Converter() Argument Clinic now emits #include "pycore_long.h" when these functions are used. --- Include/cpython/longobject.h | 4 ---- Include/internal/pycore_long.h | 7 +++++++ Modules/_blake2/clinic/blake2b_impl.c.h | 3 ++- Modules/_blake2/clinic/blake2s_impl.c.h | 3 ++- Modules/clinic/_testclinic.c.h | 3 ++- Modules/clinic/_testclinic_depr.c.h | 3 ++- Modules/clinic/overlapped.c.h | 3 ++- Modules/clinic/posixmodule.c.h | 3 ++- Modules/clinic/selectmodule.c.h | 3 ++- Modules/clinic/sha3module.c.h | 3 ++- Modules/overlapped.c | 5 ++++- PC/clinic/winreg.c.h | 3 ++- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 9 +++++++++ Tools/clinic/clinic.py | 8 ++++++++ 14 files changed, 46 insertions(+), 14 deletions(-) diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 7401566636a104..c7ae1c2cd6bc6b 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,10 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index a76494808bffdd..1dc5b7438c0879 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -186,6 +186,13 @@ extern char* _PyLong_FormatBytesWriter( int base, int alternate); +// Argument converters used by Argument Clinic +PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); + + /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. * 2: Reserved for immortality bit diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 284cccf348703c..384e7142bcfda7 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(py_blake2b_new__doc__, "blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" @@ -276,4 +277,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=76bbcf5f220511b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=48128782266b7b8e input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index 5b6af84bb24234..d7a475277e0543 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(py_blake2s_new__doc__, "blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" @@ -276,4 +277,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=af69b321be0b4a77 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2188af9910a45497 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index d3a064605f0e41..3c04c41599c650 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -3069,4 +3070,4 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=86396cbed6eb8b65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=41f439aab0cb809d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index b8365bd559b4e7..71cb972b951311 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -2391,4 +2392,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=ee8b1933e4bf4dc4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=22b59d92d517a8ec input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index b6c5f79acb1f6b..100df2062d8efb 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(_overlapped_CreateIoCompletionPort__doc__, "CreateIoCompletionPort($module, handle, port, key, concurrency, /)\n" @@ -1262,4 +1263,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=9fbc01f706562dea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=994ad727b827ff87 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index db82916b901f4b..ae3c1d60a919c1 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedInt_Converter() PyDoc_STRVAR(os_stat__doc__, "stat($module, /, path, *, dir_fd=None, follow_symlinks=True)\n" @@ -11990,4 +11991,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=5e839ce21678ea66 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a08a47b52da6da0b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index bebcc24d67e3b8..f6b3ea388108ae 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(select_select__doc__, "select($module, rlist, wlist, xlist, timeout=None, /)\n" @@ -1309,4 +1310,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=7521d757ef9e63e8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a19c29946a931dce input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 299803a3420bf6..80fe3ed32c80dc 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" @@ -193,4 +194,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=907cb475f3dc9ee0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5188d9ae4af48c6d input=a9049054013a1b77]*/ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 271f6ce7e2d916..34be3a495f3872 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -7,8 +7,11 @@ /* XXX check overflow and DWORD <-> Py_ssize_t conversions Check itemsize */ -#include "Python.h" +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif +#include "Python.h" #define WINDOWS_LEAN_AND_MEAN #include diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index a7401b9adc6880..8cb8749498e3df 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() #if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) @@ -1788,4 +1789,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=4d0ec3e43e1b28f4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=00343ee8da923da8 input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 415a2ba63dfe68..62538f53c25a2e 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -7,9 +7,18 @@ # macro. Usually it's defined by the C file which includes it. # Other header files have a similar issue. NEED_BUILD_CORE = { + # Header ".h" files 'cjkcodecs.h', 'multibytecodec.h', 'socketmodule.h', + + # Argument Clinic ".c.h" files + '_testclinic.c.h', + '_testclinic_depr.c.h', + 'overlapped.c.h', + 'posixmodule.c.h', + 'selectmodule.c.h', + 'sha3module.c.h', } TOOL = 'gcc' diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a6974bd81e2d07..c4304bb5b1270d 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3629,6 +3629,8 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'H' else: self.converter = '_PyLong_UnsignedShort_Converter' + self.add_include('pycore_long.h', + '_PyLong_UnsignedShort_Converter()') def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'H': @@ -3690,6 +3692,8 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'I' else: self.converter = '_PyLong_UnsignedInt_Converter' + self.add_include('pycore_long.h', + '_PyLong_UnsignedInt_Converter()') def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'I': @@ -3727,6 +3731,8 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'k' else: self.converter = '_PyLong_UnsignedLong_Converter' + self.add_include('pycore_long.h', + '_PyLong_UnsignedLong_Converter()') def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'k': @@ -3766,6 +3772,8 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'K' else: self.converter = '_PyLong_UnsignedLongLong_Converter' + self.add_include('pycore_long.h', + '_PyLong_UnsignedLongLong_Converter()') def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'K': From 6353c21b78a3d91e7cd7810f1c00258a34e85fe7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Aug 2023 03:18:09 +0200 Subject: [PATCH 343/598] gh-106320: Remove private _PyLong_FileDescriptor_Converter() (#108503) Move the private _PyLong converter functions to the internal C API * _PyLong_FileDescriptor_Converter(): moved to pycore_fileutils.h * _PyLong_Size_t_Converter(): moved to pycore_long.h Argument Clinic now emits includes for pycore_fileutils.h and pycore_long.h when these functions are used. --- Include/cpython/fileobject.h | 2 -- Include/cpython/longobject.h | 2 -- Include/internal/pycore_fileutils.h | 3 +++ Include/internal/pycore_long.h | 10 ++++++++++ Modules/clinic/_winapi.c.h | 3 ++- Modules/clinic/fcntlmodule.c.h | 3 ++- Modules/clinic/posixmodule.c.h | 3 ++- Modules/clinic/selectmodule.c.h | 3 ++- Modules/clinic/termios.c.h | 3 ++- Modules/fcntlmodule.c | 4 ++++ Modules/termios.c | 4 ++++ Tools/c-analyzer/c_parser/preprocessor/gcc.py | 5 ++++- Tools/clinic/clinic.py | 8 ++++++++ 13 files changed, 43 insertions(+), 10 deletions(-) diff --git a/Include/cpython/fileobject.h b/Include/cpython/fileobject.h index b70ec318986d82..6fbe69604af40c 100644 --- a/Include/cpython/fileobject.h +++ b/Include/cpython/fileobject.h @@ -15,5 +15,3 @@ typedef PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *); PyAPI_FUNC(PyObject *) PyFile_OpenCode(const char *utf8path); PyAPI_FUNC(PyObject *) PyFile_OpenCodeObject(PyObject *path); PyAPI_FUNC(int) PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData); - -PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index c7ae1c2cd6bc6b..32c40b083b7ab6 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,8 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); - PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); /* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 90c0878a3ee7d6..f6e625870de9d1 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -312,6 +312,9 @@ extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootE # define _Py_END_SUPPRESS_IPH #endif /* _MSC_VER >= 1900 */ +// Export for 'select' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 1dc5b7438c0879..c9d82711862895 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -187,11 +187,21 @@ extern char* _PyLong_FormatBytesWriter( int alternate); // Argument converters used by Argument Clinic + +// Export for 'select' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); + +// Export for '_testclinic' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); + +// Export for '_blake2' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); + +// Export for '_blake2' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); +// Export for '_testclinic' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 35ac053547121c..c648e68be448c6 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_Size_t_Converter() PyDoc_STRVAR(_winapi_Overlapped_GetOverlappedResult__doc__, "GetOverlappedResult($self, wait, /)\n" @@ -1478,4 +1479,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=ff91ab5cae8961dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c7e08927e163ef13 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index bd978b63e4dd6b..c15f345ee6ffe8 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" @@ -249,4 +250,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=705976d5f53f2272 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f096e8731fa38be input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ae3c1d60a919c1..4b85519b532137 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() PyDoc_STRVAR(os_stat__doc__, @@ -11991,4 +11992,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a08a47b52da6da0b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ff0e50316f4ed71a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index f6b3ea388108ae..69c0f06789e714 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(select_select__doc__, @@ -1310,4 +1311,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=a19c29946a931dce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a215af2157f038c7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 4e911aa0a5fb9b..44d4107c378e53 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() PyDoc_STRVAR(termios_tcgetattr__doc__, "tcgetattr($module, fd, /)\n" @@ -292,4 +293,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=4c79a3bf87370275 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=434df4394b596e92 input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index e530621fd269a1..3bf5830b2edca9 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,5 +1,9 @@ /* fcntl module */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #ifdef HAVE_SYS_FILE_H diff --git a/Modules/termios.c b/Modules/termios.c index 6b254104ed1e21..21d3541c177bc3 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1,5 +1,9 @@ /* termios.c -- POSIX terminal I/O module implementation. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" /* Apparently, on SGI, termios.h won't define CTRL if _XOPEN_SOURCE diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 62538f53c25a2e..de9a2484de82bb 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -12,13 +12,16 @@ 'multibytecodec.h', 'socketmodule.h', - # Argument Clinic ".c.h" files + # Argument Clinic ".c.h" header files '_testclinic.c.h', '_testclinic_depr.c.h', + '_winapi.c.h', + 'fcntlmodule.c.h', 'overlapped.c.h', 'posixmodule.c.h', 'selectmodule.c.h', 'sha3module.c.h', + 'termios.c.h', } TOOL = 'gcc' diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c4304bb5b1270d..e622d254ee8ccb 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3835,6 +3835,10 @@ class size_t_converter(CConverter): converter = '_PyLong_Size_t_Converter' c_ignored_default = "0" + def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: + self.add_include('pycore_long.h', + '_PyLong_Size_t_Converter()') + def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'n': return """ @@ -3850,6 +3854,10 @@ class fildes_converter(CConverter): type = 'int' converter = '_PyLong_FileDescriptor_Converter' + def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: + self.add_include('pycore_fileutils.h', + '_PyLong_FileDescriptor_Converter()') + def _parse_arg(self, argname: str, displayname: str) -> str | None: return """ {paramname} = PyObject_AsFileDescriptor({argname}); From 8ba47146111d714c7b61825d43b52311d9be366d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Aug 2023 04:05:17 +0200 Subject: [PATCH 344/598] gh-106320: Remove private AC converter functions (#108505) Move these private functions to the internal C API (pycore_abstract.h): * _Py_convert_optional_to_ssize_t() * _PyNumber_Index() Argument Clinic now emits #include "pycore_abstract.h" when these functions are used. The parser of the c-analyzer tool now uses a list of files which use the limited C API, rather than a list of files using the internal C API. --- Include/cpython/abstract.h | 10 ------ Include/internal/pycore_abstract.h | 7 +++++ Modules/_io/_iomodule.c | 1 + Modules/_io/clinic/bufferedio.c.h | 3 +- Modules/_io/clinic/bytesio.c.h | 3 +- Modules/_io/clinic/fileio.c.h | 3 +- Modules/_io/clinic/iobase.c.h | 3 +- Modules/_io/clinic/stringio.c.h | 3 +- Modules/_io/clinic/textio.c.h | 3 +- Modules/_io/clinic/winconsoleio.c.h | 3 +- Modules/_sre/clinic/sre.c.h | 3 +- Modules/clinic/_bisectmodule.c.h | 3 +- Modules/clinic/_bz2module.c.h | 3 +- Modules/clinic/_collectionsmodule.c.h | 3 +- Modules/clinic/_elementtree.c.h | 3 +- Modules/clinic/_hashopenssl.c.h | 3 +- Modules/clinic/_lzmamodule.c.h | 3 +- Modules/clinic/_operator.c.h | 3 +- Modules/clinic/_ssl.c.h | 3 +- Modules/clinic/_struct.c.h | 3 +- Modules/clinic/_testclinic.c.h | 3 +- Modules/clinic/_testclinic_depr.c.h | 3 +- Modules/clinic/arraymodule.c.h | 3 +- Modules/clinic/gcmodule.c.h | 3 +- Modules/clinic/itertoolsmodule.c.h | 3 +- Modules/clinic/posixmodule.c.h | 3 +- Modules/clinic/zlibmodule.c.h | 3 +- Modules/mathmodule.c | 1 + Modules/mmapmodule.c | 1 + Modules/posixmodule.c | 1 + Objects/clinic/bytearrayobject.c.h | 3 +- Objects/clinic/bytesobject.c.h | 3 +- Objects/clinic/listobject.c.h | 3 +- Objects/clinic/longobject.c.h | 3 +- Objects/clinic/unicodeobject.c.h | 3 +- Objects/floatobject.c | 1 + Objects/stringlib/clinic/transmogrify.h.h | 3 +- Python/clinic/_warnings.c.h | 3 +- Python/getargs.c | 1 + Tools/c-analyzer/c_parser/preprocessor/gcc.py | 31 ++++++------------- Tools/clinic/clinic.py | 3 ++ 41 files changed, 88 insertions(+), 62 deletions(-) diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index dd924dfd3d8fcc..1f495f19df280b 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -53,13 +53,3 @@ PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t); need to be corrected for a negative index. */ #define PySequence_ITEM(o, i)\ ( Py_TYPE(o)->tp_as_sequence->sq_item((o), (i)) ) - -/* === Mapping protocol ================================================= */ - -// Convert Python int to Py_ssize_t. Do nothing if the argument is None. -// Cannot be moved to the internal C API: used by Argument Clinic. -PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); - -// Same as PyNumber_Index but can return an instance of a subclass of int. -// Cannot be moved to the internal C API: used by Argument Clinic. -PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 2733d8102e5ef4..3cc0afac4bd5b4 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -47,6 +47,13 @@ extern int _PyObject_RealIsInstance(PyObject *inst, PyObject *cls); extern int _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); +// Convert Python int to Py_ssize_t. Do nothing if the argument is None. +// Export for '_bisect' shared extension. +PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); + +// Same as PyNumber_Index() but can return an instance of a subclass of int. +// Export for 'math' shared extension. +PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); #ifdef __cplusplus } diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index f56654227f1020..0762e26a36c795 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -8,6 +8,7 @@ */ #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index b9b42cf3428573..32c4a4e177cbbe 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_io__BufferedIOBase_readinto__doc__, "readinto($self, buffer, /)\n" @@ -1098,4 +1099,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e5b335441452653d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=768c3a3a3deabcb4 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 52c9c2c5ecbb6e..b5cdd0802b8748 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io_BytesIO_readable__doc__, "readable($self, /)\n" @@ -538,4 +539,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8d4e7651002e14c6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b1219bda0619e2a input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 9f5198d45ab819..7b63e36b5038db 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io_FileIO_close__doc__, "close($self, /)\n" @@ -536,4 +537,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=65b9a5cc96d193b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3a3c6ed7e5e78063 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index a539125a2a8fce..f582b9d2c71042 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io__IOBase_seek__doc__, "seek($self, offset, whence=os.SEEK_SET, /)\n" @@ -436,4 +437,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=0f064cfd54e3c1a5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ec741e0961671a86 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index d0acfdb4bf3818..dbe1833a90ed8c 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io_StringIO_getvalue__doc__, "getvalue($self, /)\n" @@ -367,4 +368,4 @@ _io_StringIO_seekable(stringio *self, PyObject *Py_UNUSED(ignored)) { return _io_StringIO_seekable_impl(self); } -/*[clinic end generated code: output=6f55dc1454aeb507 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=09d4056cc8c4aae4 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index d653cc87ae317a..03d5facedfb513 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "detach($self, /)\n" @@ -975,4 +976,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=29b945b24287dd0c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c3a8eb2591be1bf7 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index cd3348dc1227cd..53f971e37c1258 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #if defined(HAVE_WINDOWS_CONSOLE_IO) @@ -465,4 +466,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=235393758365c229 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0cdb16d95f1c7dac input=a9049054013a1b77]*/ diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index a303a557ca5f46..1846f175b4585a 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_sre_getcodesize__doc__, "getcodesize($module, /)\n" @@ -1460,4 +1461,4 @@ _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=6b84e62c87238f0e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b8cf77f05e44d08c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 7944f5219b02a3..62ce0fc8172367 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_bisect_bisect_right__doc__, "bisect_right($module, /, a, x, lo=0, hi=None, *, key=None)\n" @@ -433,4 +434,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=5a7fa64bf9b262f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=839fdddeacdc2ecb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 6ad2db3ce1dee6..7c4d57b5fec7e1 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_bz2_BZ2Compressor_compress__doc__, "compress($self, data, /)\n" @@ -241,4 +242,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=431fd0fc40f019d1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=40706688f92642b4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index 3882d069216e28..b96502d3cf4b3f 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_collections__count_elements__doc__, "_count_elements($module, mapping, iterable, /)\n" @@ -75,4 +76,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=00e516317d2b8bed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d83188040656ad7c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index a0bc751ca21ee8..78e456b6a43463 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_elementtree_Element_append__doc__, "append($self, subelement, /)\n" @@ -1218,4 +1219,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=40767b1a98e54b60 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a34620406b95eb0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index fb61a444018dbb..0ea192eac2d3f1 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(EVP_copy__doc__, "copy($self, /)\n" @@ -1851,4 +1852,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=b339e255db698147 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cfad8d5e904a4917 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 017b24f765eee3..ceb75399157627 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_lzma_LZMACompressor_compress__doc__, "compress($self, data, /)\n" @@ -338,4 +339,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=aaf225a5d15d3e75 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f1a001f5f489c372 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_operator.c.h b/Modules/clinic/_operator.c.h index b68e6e0144a586..18b0b920b0d118 100644 --- a/Modules/clinic/_operator.c.h +++ b/Modules/clinic/_operator.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_operator_truth__doc__, "truth($module, a, /)\n" @@ -1492,4 +1493,4 @@ _operator__compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=227cbcfed44f736e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=72bc63a775937245 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 81712617bee12b..6ab8e3ee436336 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, "do_handshake($self, /)\n" @@ -1542,4 +1543,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=4c9f00c62f0825d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6e2eb86330f3f9b8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index c3cf179ed4c040..a823364447818b 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(Struct__doc__, "Struct(format)\n" @@ -451,4 +452,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=f3d6e06f80368998 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9b48aeaa86898ec5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 3c04c41599c650..94c4b5a1221eec 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(test_empty_function__doc__, @@ -3070,4 +3071,4 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=41f439aab0cb809d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a652e6b1787d3346 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 71cb972b951311..7b9d8971efb46b 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(depr_star_new__doc__, @@ -2392,4 +2393,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=22b59d92d517a8ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=464aeba97e482f5c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 5e957013ad0dc3..8bdfcc62bef085 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(array_array___copy____doc__, "__copy__($self, /)\n" @@ -674,4 +675,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=db417f5677a25eaa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab24c7a40a41c2e1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 0ea5b40cfb770a..290a8aaa2728e5 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -424,4 +425,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=16003cfbc66ada39 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fe89ce0b90a1067d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 32278bf715aa98..6555bd495b200e 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(batched_new__doc__, "batched(iterable, n)\n" @@ -913,4 +914,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=111cbd102c2a23c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=20999801c7349d2c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 4b85519b532137..e80bbff6b5156d 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() @@ -11992,4 +11993,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=ff0e50316f4ed71a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b34619e5f65adc2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 3d71536c846109..1debe2ae98959d 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(zlib_compress__doc__, "compress($module, data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS)\n" @@ -1129,4 +1130,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=3eccb3f7265d53ba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d574a79aa47c9969 input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 0d238c086ac78f..d929dcf65a7e32 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -57,6 +57,7 @@ raised for division by zero and mod by zero. #endif #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5c131570123560..44d5b268e1b74f 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -23,6 +23,7 @@ #endif #include +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cffa401d2ceb02..8eb25fc242495c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12,6 +12,7 @@ #ifdef __VXWORKS__ # include "pycore_bitutils.h" // _Py_popcount32() #endif +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_fileutils.h" // _Py_closerange() diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index ecb08a86c2aab5..f091fc7afbd693 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() static int bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, @@ -1284,4 +1285,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=cb94d7ac45f0a4b7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d388e9027b333f00 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 904ff07e97cde3..66db76dbbb4faf 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(bytes___bytes____doc__, "__bytes__($self, /)\n" @@ -1060,4 +1061,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=f4e85363c79c64d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9da56b6c04914e18 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index e3d6ffa9f76fdb..a41ed3e4b040d9 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(list_insert__doc__, "insert($self, index, object, /)\n" @@ -383,4 +384,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=2ca109d8acc775bc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=537a17b562c57505 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index d37e44f2f330bd..3ecef1f0fd54d4 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() static PyObject * long_new_impl(PyTypeObject *type, PyObject *x, PyObject *obase); @@ -475,4 +476,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=75ed306fff493ba1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ea9c87ea532dadbe input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 7edb4bc76099be..56b64087d6654a 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(EncodingMap_size__doc__, "size($self, /)\n" @@ -1504,4 +1505,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=2e1e1b136ef1b681 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a64776a3ea1c970c input=a9049054013a1b77]*/ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 6a0c2e033e3e9a..1c5078bdda6871 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -4,6 +4,7 @@ for any kind of float exception without losing portability. */ #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() #include "pycore_initconfig.h" // _PyStatus_OK() diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index 1e0df0958d6a4d..d1adda4e979c14 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(stringlib_expandtabs__doc__, "expandtabs($self, /, tabsize=8)\n" @@ -278,4 +279,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=00e34c03331699fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a0338b2d41671b17 input=a9049054013a1b77]*/ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index b39e412a7d6de9..1a01416e7363c7 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(warnings_warn__doc__, "warn($module, /, message, category=None, stacklevel=1, source=None, *,\n" @@ -243,4 +244,4 @@ warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) { return warnings_filters_mutated_impl(module); } -/*[clinic end generated code: output=0e2b367a662bf51b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c8a6dc1403fba1d5 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index 809b2d4ef17d3d..fdc144488c9627 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2,6 +2,7 @@ /* New getargs implementation */ #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dict.h" // _PyDict_HasOnlyStringKeys() #include "pycore_pylifecycle.h" // _PyArg_Fini #include "pycore_tuple.h" // _PyTuple_ITEMS() diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index de9a2484de82bb..55cc2d37e1ebc7 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -3,26 +3,14 @@ from . import common as _common -# Modules/socketmodule.h uses pycore_time.h which needs the Py_BUILD_CORE -# macro. Usually it's defined by the C file which includes it. -# Other header files have a similar issue. -NEED_BUILD_CORE = { - # Header ".h" files - 'cjkcodecs.h', - 'multibytecodec.h', - 'socketmodule.h', - - # Argument Clinic ".c.h" header files - '_testclinic.c.h', - '_testclinic_depr.c.h', - '_winapi.c.h', - 'fcntlmodule.c.h', - 'overlapped.c.h', - 'posixmodule.c.h', - 'selectmodule.c.h', - 'sha3module.c.h', - 'termios.c.h', -} +# The following C files must not be built with Py_BUILD_CORE, +# because they use the limited C API. +USE_LIMITED_C_API = frozenset(( + '_testcapimodule.c', + '_testclinic_limited.c', + 'xxlimited.c', + 'xxlimited_35.c', +)) TOOL = 'gcc' @@ -81,8 +69,9 @@ def preprocess(filename, cwd = os.path.abspath(cwd or '.') filename = _normpath(filename, cwd) + print(filename) postargs = POST_ARGS - if os.path.basename(filename) in NEED_BUILD_CORE: + if os.path.basename(filename) not in USE_LIMITED_C_API: postargs += ('-DPy_BUILD_CORE=1',) text = _common.preprocess( diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index e622d254ee8ccb..70ec18f726e3f4 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3795,8 +3795,11 @@ def converter_init(self, *, accept: TypeSet = {int}) -> None: if accept == {int}: self.format_unit = 'n' self.default_type = int + self.add_include('pycore_abstract.h', '_PyNumber_Index()') elif accept == {int, NoneType}: self.converter = '_Py_convert_optional_to_ssize_t' + self.add_include('pycore_abstract.h', + '_Py_convert_optional_to_ssize_t()') else: fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") From e407cea1938b80b1d469f148a4ea65587820e3eb Mon Sep 17 00:00:00 2001 From: denballakh <47365157+denballakh@users.noreply.github.com> Date: Sat, 26 Aug 2023 14:54:16 +0500 Subject: [PATCH 345/598] gh-107406: Add better `struct.Struct` repr (#107407) --- Doc/library/struct.rst | 9 +++++++++ Lib/test/test_struct.py | 4 ++++ .../2023-07-29-02-01-24.gh-issue-107406.ze6sQP.rst | 2 ++ Modules/_struct.c | 14 ++++++++++++++ 4 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-29-02-01-24.gh-issue-107406.ze6sQP.rst diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index c94dfde4d55763..e2e6fc542e3e67 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -1,6 +1,10 @@ :mod:`struct` --- Interpret bytes as packed binary data ======================================================= +.. testsetup:: * + + from struct import * + .. module:: struct :synopsis: Interpret bytes as packed binary data. @@ -597,6 +601,11 @@ The :mod:`struct` module also defines the following type: The calculated size of the struct (and hence of the bytes object produced by the :meth:`pack` method) corresponding to :attr:`format`. + .. versionchanged:: 3.13 The *repr()* of structs has changed. It + is now: + + >>> Struct('i') + Struct('i') .. _half precision format: https://en.wikipedia.org/wiki/Half-precision_floating-point_format diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 6b1f22f66fd157..c76649cdcd9cce 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -774,6 +774,10 @@ def test_error_propagation(fmt_str): test_error_propagation('N') test_error_propagation('n') + def test_repr(self): + s = struct.Struct('=i2H') + self.assertEqual(repr(s), f'Struct({s.format!r})') + class UnpackIteratorTest(unittest.TestCase): """ Tests for iterative unpacking (struct.Struct.iter_unpack). diff --git a/Misc/NEWS.d/next/Library/2023-07-29-02-01-24.gh-issue-107406.ze6sQP.rst b/Misc/NEWS.d/next/Library/2023-07-29-02-01-24.gh-issue-107406.ze6sQP.rst new file mode 100644 index 00000000000000..b78e57f591e78c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-29-02-01-24.gh-issue-107406.ze6sQP.rst @@ -0,0 +1,2 @@ +Implement new :meth:`__repr__` method for :class:`struct.Struct`. +Now it returns ``Struct()``. diff --git a/Modules/_struct.c b/Modules/_struct.c index 606ae5e4e19964..4da654231cdf00 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2165,6 +2165,19 @@ s_sizeof(PyStructObject *self, void *unused) return PyLong_FromSize_t(size); } +static PyObject * +s_repr(PyStructObject *self) +{ + PyObject* fmt = PyUnicode_FromStringAndSize( + PyBytes_AS_STRING(self->s_format), PyBytes_GET_SIZE(self->s_format)); + if (fmt == NULL) { + return NULL; + } + PyObject* s = PyUnicode_FromFormat("%s(%R)", _PyType_Name(Py_TYPE(self)), fmt); + Py_DECREF(fmt); + return s; +} + /* List of functions */ static struct PyMethodDef s_methods[] = { @@ -2197,6 +2210,7 @@ static PyType_Slot PyStructType_slots[] = { {Py_tp_dealloc, s_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_setattro, PyObject_GenericSetAttr}, + {Py_tp_repr, s_repr}, {Py_tp_doc, (void*)s__doc__}, {Py_tp_traverse, s_traverse}, {Py_tp_clear, s_clear}, From 2b15536fa94d07e9e286826c23507402313ec7f4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Aug 2023 00:35:06 +0300 Subject: [PATCH 346/598] gh-107913: Fix possible losses of OSError error codes (GH-107930) Functions like PyErr_SetFromErrno() and SetFromWindowsErr() should be called immediately after using the C API which sets errno or the Windows error code. --- ...-08-14-11-18-13.gh-issue-107913.4ooY6i.rst | 3 + Modules/_cursesmodule.c | 2 +- Modules/_io/fileio.c | 12 +- Modules/_io/winconsoleio.c | 2 +- Modules/_localemodule.c | 2 +- Modules/_multiprocessing/semaphore.c | 9 +- Modules/_ssl.c | 8 +- Modules/faulthandler.c | 3 +- Modules/fcntlmodule.c | 5 +- Modules/getpath.c | 3 +- Modules/mmapmodule.c | 2 + Modules/overlapped.c | 3 +- Modules/posixmodule.c | 128 ++++++++++++------ Modules/selectmodule.c | 4 +- Modules/socketmodule.c | 12 +- Objects/unicodeobject.c | 2 +- Python/fileutils.c | 4 +- 17 files changed, 129 insertions(+), 75 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-14-11-18-13.gh-issue-107913.4ooY6i.rst diff --git a/Misc/NEWS.d/next/Library/2023-08-14-11-18-13.gh-issue-107913.4ooY6i.rst b/Misc/NEWS.d/next/Library/2023-08-14-11-18-13.gh-issue-107913.4ooY6i.rst new file mode 100644 index 00000000000000..de5e21abfc3ae7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-11-18-13.gh-issue-107913.4ooY6i.rst @@ -0,0 +1,3 @@ +Fix possible losses of ``errno`` and ``winerror`` values in :exc:`OSError` +exceptions if they were cleared or modified by the cleanup code before +creating the exception object. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 1f5afa6fcd898d..d339a8aa798361 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3069,8 +3069,8 @@ _curses_getwin(PyObject *module, PyObject *file) } datalen = PyBytes_GET_SIZE(data); if (fwrite(PyBytes_AS_STRING(data), 1, datalen, fp) != datalen) { - Py_DECREF(data); PyErr_SetFromErrno(PyExc_OSError); + Py_DECREF(data); goto error; } Py_DECREF(data); diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index d52bcd50bd28c0..15df1b2befda82 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -393,6 +393,11 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (async_err) goto error; + + if (self->fd < 0) { + PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); + goto error; + } } else { PyObject *fdobj; @@ -424,12 +429,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, goto error; } } - fd_is_own = 1; - if (self->fd < 0) { - PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); - goto error; - } #ifndef MS_WINDOWS if (_Py_set_inheritable(self->fd, 0, atomic_flag_works) < 0) @@ -1057,8 +1057,8 @@ _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) Py_END_ALLOW_THREADS if (ret != 0) { - Py_DECREF(posobj); PyErr_SetFromErrno(PyExc_OSError); + Py_DECREF(posobj); return NULL; } diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 875c90c2ccd7bb..e10a22cdeb678d 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -377,8 +377,8 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, else self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY); if (self->fd < 0) { - CloseHandle(handle); PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); + CloseHandle(handle); goto error; } } diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 34915de7515d63..e4b956b136dd1d 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -736,8 +736,8 @@ _locale_bindtextdomain_impl(PyObject *module, const char *domain, } current_dirname = bindtextdomain(domain, dirname); if (current_dirname == NULL) { - Py_XDECREF(dirname_bytes); PyErr_SetFromErrno(PyExc_OSError); + Py_XDECREF(dirname_bytes); return NULL; } result = PyUnicode_DecodeLocale(current_dirname, NULL); diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index 771f86e5367af3..d22b8d18e33e42 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -516,12 +516,12 @@ _multiprocessing_SemLock_impl(PyTypeObject *type, int kind, int value, return result; failure: - if (handle != SEM_FAILED) - SEM_CLOSE(handle); - PyMem_Free(name_copy); if (!PyErr_Occurred()) { _PyMp_SetError(NULL, MP_STANDARD_ERROR); } + if (handle != SEM_FAILED) + SEM_CLOSE(handle); + PyMem_Free(name_copy); return NULL; } @@ -556,8 +556,9 @@ _multiprocessing_SemLock__rebuild_impl(PyTypeObject *type, SEM_HANDLE handle, if (name != NULL) { handle = sem_open(name, 0); if (handle == SEM_FAILED) { + PyErr_SetFromErrno(PyExc_OSError); PyMem_Free(name_copy); - return PyErr_SetFromErrno(PyExc_OSError); + return NULL; } } #endif diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c001b875906d44..be033256adc259 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -3914,8 +3914,8 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, /* the password callback has already set the error information */ } else if (errno != 0) { - ERR_clear_error(); PyErr_SetFromErrno(PyExc_OSError); + ERR_clear_error(); } else { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); @@ -3935,8 +3935,8 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, /* the password callback has already set the error information */ } else if (errno != 0) { - ERR_clear_error(); PyErr_SetFromErrno(PyExc_OSError); + ERR_clear_error(); } else { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); @@ -4165,8 +4165,8 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, PySSL_END_ALLOW_THREADS if (r != 1) { if (errno != 0) { - ERR_clear_error(); PyErr_SetFromErrno(PyExc_OSError); + ERR_clear_error(); } else { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); @@ -4213,8 +4213,8 @@ _ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) PySSL_END_ALLOW_THREADS if (dh == NULL) { if (errno != 0) { - ERR_clear_error(); PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath); + ERR_clear_error(); } else { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index aecb64dbb44647..f05cdd9a37f8a4 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -415,11 +415,10 @@ faulthandler_allocate_stack(void) int err = sigaltstack(&stack, &old_stack); if (err) { + PyErr_SetFromErrno(PyExc_OSError); /* Release the stack to retry sigaltstack() next time */ PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; - - PyErr_SetFromErrno(PyExc_OSError); return -1; } return 0; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 3bf5830b2edca9..fd03abf0561da6 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -212,11 +212,12 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, if (mutate_arg && (len <= IOCTL_BUFSZ)) { memcpy(str, buf, len); } - PyBuffer_Release(&pstr); /* No further access to str below this point */ if (ret < 0) { PyErr_SetFromErrno(PyExc_OSError); + PyBuffer_Release(&pstr); return NULL; } + PyBuffer_Release(&pstr); if (mutate_arg) { return PyLong_FromLong(ret); } @@ -241,8 +242,8 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, ret = ioctl(fd, code, buf); Py_END_ALLOW_THREADS if (ret < 0) { - PyBuffer_Release(&pstr); PyErr_SetFromErrno(PyExc_OSError); + PyBuffer_Release(&pstr); return NULL; } PyBuffer_Release(&pstr); diff --git a/Modules/getpath.c b/Modules/getpath.c index fb1656ada01496..71e23e1edaf11c 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -342,11 +342,12 @@ getpath_readlines(PyObject *Py_UNUSED(self), PyObject *args) return NULL; } FILE *fp = _Py_wfopen(path, L"rb"); - PyMem_Free((void *)path); if (!fp) { PyErr_SetFromErrno(PyExc_OSError); + PyMem_Free((void *)path); return NULL; } + PyMem_Free((void *)path); r = PyList_New(0); if (!r) { diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 44d5b268e1b74f..c8cd7e59dbab50 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1356,6 +1356,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->data = mmap(NULL, map_size, prot, flags, fd, offset); Py_END_ALLOW_THREADS + int saved_errno = errno; if (devzero != -1) { close(devzero); } @@ -1363,6 +1364,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (m_obj->data == (char *)-1) { m_obj->data = NULL; Py_DECREF(m_obj); + errno = saved_errno; PyErr_SetFromErrno(PyExc_OSError); return NULL; } diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 34be3a495f3872..c682e6adccc765 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -370,8 +370,9 @@ _overlapped_RegisterWaitWithQueue_impl(PyObject *module, HANDLE Object, &NewWaitObject, Object, PostToQueueCallback, pdata, Milliseconds, WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE)) { + SetFromWindowsErr(0); PyMem_RawFree(pdata); - return SetFromWindowsErr(0); + return NULL; } return Py_BuildValue(F_HANDLE, NewWaitObject); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8eb25fc242495c..0436571abc9054 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3877,9 +3877,10 @@ posix_getcwd(int use_bytes) return NULL; } if (!len) { + PyErr_SetFromWindowsErr(0); if (wbuf2 != wbuf) PyMem_RawFree(wbuf2); - return PyErr_SetFromWindowsErr(0); + return NULL; } PyObject *resobj = PyUnicode_FromWideChar(wbuf2, len); @@ -3927,8 +3928,9 @@ posix_getcwd(int use_bytes) return PyErr_NoMemory(); } if (cwd == NULL) { + posix_error(); PyMem_RawFree(buf); - return posix_error(); + return NULL; } PyObject *obj; @@ -4140,8 +4142,8 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) int error = GetLastError(); if (error == ERROR_FILE_NOT_FOUND) goto exit; - Py_DECREF(list); - list = path_error(path); + path_error(path); + Py_CLEAR(list); goto exit; } do { @@ -4154,12 +4156,12 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) Py_SETREF(v, PyUnicode_EncodeFSDefault(v)); } if (v == NULL) { - Py_SETREF(list, NULL); + Py_CLEAR(list); break; } if (PyList_Append(list, v) != 0) { Py_DECREF(v); - Py_SETREF(list, NULL); + Py_CLEAR(list); break; } Py_DECREF(v); @@ -4170,8 +4172,8 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) /* FindNextFile sets error to ERROR_NO_MORE_FILES if it got to the end of the directory. */ if (!result && GetLastError() != ERROR_NO_MORE_FILES) { - Py_DECREF(list); - list = path_error(path); + path_error(path); + Py_CLEAR(list); goto exit; } } while (result == TRUE); @@ -4180,8 +4182,8 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (hFindFile != INVALID_HANDLE_VALUE) { if (FindClose(hFindFile) == FALSE) { if (list != NULL) { - Py_DECREF(list); - list = path_error(path); + path_error(path); + Py_CLEAR(list); } } } @@ -4243,7 +4245,8 @@ _posix_listdir(path_t *path, PyObject *list) } if (dirp == NULL) { - list = path_error(path); + path_error(path); + list = NULL; #ifdef HAVE_FDOPENDIR if (fd != -1) { Py_BEGIN_ALLOW_THREADS @@ -4265,8 +4268,8 @@ _posix_listdir(path_t *path, PyObject *list) if (errno == 0) { break; } else { - Py_DECREF(list); - list = path_error(path); + path_error(path); + Py_CLEAR(list); goto exit; } } @@ -6609,8 +6612,9 @@ os_execv_impl(PyObject *module, path_t *path, PyObject *argv) /* If we get here it's definitely an error */ + posix_error(); free_string_array(argvlist, argc); - return posix_error(); + return NULL; } @@ -6914,11 +6918,12 @@ parse_file_actions(PyObject *file_actions, } errno = posix_spawn_file_actions_addopen(file_actionsp, fd, PyBytes_AS_STRING(path), oflag, (mode_t)mode); - Py_DECREF(path); if (errno) { posix_error(); + Py_DECREF(path); goto fail; } + Py_DECREF(path); break; } case POSIX_SPAWN_CLOSE: { @@ -7315,12 +7320,15 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS + int saved_errno = errno; free_string_array(argvlist, argc); - if (spawnval == -1) - return posix_error(); - else - return Py_BuildValue(_Py_PARSE_INTPTR, spawnval); + if (spawnval == -1) { + errno = saved_errno; + posix_error(); + return NULL; + } + return Py_BuildValue(_Py_PARSE_INTPTR, spawnval); } /*[clinic input] @@ -7638,6 +7646,7 @@ os_fork1_impl(PyObject *module) } PyOS_BeforeFork(); pid = fork1(); + int saved_errno = errno; if (pid == 0) { /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); @@ -7646,8 +7655,10 @@ os_fork1_impl(PyObject *module) /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } - if (pid == -1) + if (pid == -1) { + errno = saved_errno; return posix_error(); + } return PyLong_FromPid(pid); } #endif /* HAVE_FORK1 */ @@ -7683,6 +7694,7 @@ os_fork_impl(PyObject *module) } PyOS_BeforeFork(); pid = fork(); + int saved_errno = errno; if (pid == 0) { /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); @@ -7691,8 +7703,10 @@ os_fork_impl(PyObject *module) /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } - if (pid == -1) + if (pid == -1) { + errno = saved_errno; return posix_error(); + } return PyLong_FromPid(pid); } #endif /* HAVE_FORK */ @@ -8229,13 +8243,17 @@ os_openpty_impl(PyObject *module) /* change permission of slave */ if (grantpt(master_fd) < 0) { + int saved_errno = errno; PyOS_setsig(SIGCHLD, sig_saved); + errno = saved_errno; goto posix_error; } /* unlock slave */ if (unlockpt(master_fd) < 0) { + int saved_errno = errno; PyOS_setsig(SIGCHLD, sig_saved); + errno = saved_errno; goto posix_error; } @@ -8599,8 +8617,9 @@ os_getgroups_impl(PyObject *module) n = getgroups(n, grouplist); if (n == -1) { + posix_error(); PyMem_Free(grouplist); - return posix_error(); + return NULL; } PyObject *result = PyList_New(n); @@ -9167,8 +9186,9 @@ os_setgroups(PyObject *module, PyObject *groups) } if (setgroups(len, grouplist) < 0) { + posix_error(); PyMem_Free(grouplist); - return posix_error(); + return NULL; } PyMem_Free(grouplist); Py_RETURN_NONE; @@ -10615,10 +10635,13 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers) Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + int saved_errno = errno; iov_cleanup(iov, buf, cnt); if (n < 0) { - if (!async_err) + if (!async_err) { + errno = saved_errno; posix_error(); + } return -1; } @@ -10667,8 +10690,11 @@ os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (n < 0) { + if (!async_err) { + posix_error(); + } Py_DECREF(buffer); - return (!async_err) ? posix_error() : NULL; + return NULL; } if (n != length) _PyBytes_Resize(&buffer, n); @@ -10765,9 +10791,11 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, #endif + int saved_errno = errno; iov_cleanup(iov, buf, cnt); if (n < 0) { if (!async_err) { + errno = saved_errno; posix_error(); } return -1; @@ -10936,24 +10964,26 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); _Py_END_SUPPRESS_IPH + int saved_errno = errno; if (sf.headers != NULL) iov_cleanup(sf.headers, hbuf, sf.hdr_cnt); if (sf.trailers != NULL) iov_cleanup(sf.trailers, tbuf, sf.trl_cnt); if (ret < 0) { - if ((errno == EAGAIN) || (errno == EBUSY)) { + if ((saved_errno == EAGAIN) || (saved_errno == EBUSY)) { if (sbytes != 0) { // some data has been sent goto done; } - else { - // no data has been sent; upper application is supposed - // to retry on EAGAIN or EBUSY - return posix_error(); - } + // no data has been sent; upper application is supposed + // to retry on EAGAIN or EBUSY } - return (!async_err) ? posix_error() : NULL; + if (!async_err) { + errno = saved_errno; + posix_error(); + } + return NULL; } goto done; @@ -11270,10 +11300,10 @@ os_writev_impl(PyObject *module, int fd, PyObject *buffers) Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - iov_cleanup(iov, buf, cnt); if (result < 0 && !async_err) posix_error(); + iov_cleanup(iov, buf, cnt); return result; } #endif /* HAVE_WRITEV */ @@ -11408,13 +11438,13 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, #endif - iov_cleanup(iov, buf, cnt); if (result < 0) { if (!async_err) { posix_error(); } - return -1; + result = -1; } + iov_cleanup(iov, buf, cnt); return result; } @@ -11976,12 +12006,13 @@ win32_putenv(PyObject *name, PyObject *value) Prefer _wputenv() to be compatible with C libraries using CRT variables and CRT functions using these variables (ex: getenv()). */ int err = _wputenv(env); - PyMem_Free(env); if (err) { posix_error(); + PyMem_Free(env); return NULL; } + PyMem_Free(env); Py_RETURN_NONE; } @@ -13823,10 +13854,12 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_END_ALLOW_THREADS; if (result < 0) { - Py_DECREF(buffer); - if (errno == ERANGE) + if (errno == ERANGE) { + Py_DECREF(buffer); continue; + } path_error(path); + Py_DECREF(buffer); return NULL; } @@ -14593,14 +14626,19 @@ DirEntry_fetch_stat(PyObject *module, DirEntry *self, int follow_symlinks) } Py_END_ALLOW_THREADS } + + int saved_errno = errno; #if defined(MS_WINDOWS) PyMem_Free(path); #else Py_DECREF(ub); #endif - if (result != 0) - return path_object_error(self->path); + if (result != 0) { + errno = saved_errno; + path_object_error(self->path); + return NULL; + } return _pystat_fromstructstat(module, &st); } @@ -14792,10 +14830,14 @@ os_DirEntry_inode_impl(DirEntry *self) wchar_t *path = PyUnicode_AsWideCharString(unicode, NULL); Py_DECREF(unicode); result = LSTAT(path, &stat); + + int saved_errno = errno; PyMem_Free(path); - if (result != 0) + if (result != 0) { + errno = saved_errno; return path_object_error(self->path); + } self->win32_file_index = stat.st_ino; self->got_file_index = 1; @@ -15359,12 +15401,12 @@ os_scandir_impl(PyObject *module, path_t *path) iterator->handle = FindFirstFileW(path_strW, &iterator->file_data); Py_END_ALLOW_THREADS - PyMem_Free(path_strW); - if (iterator->handle == INVALID_HANDLE_VALUE) { path_error(&iterator->path); + PyMem_Free(path_strW); goto error; } + PyMem_Free(path_strW); #else /* POSIX */ errno = 0; #ifdef HAVE_FDOPENDIR diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 94d246960f4410..4987cf0f2065c2 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1294,8 +1294,8 @@ newPyEpoll_Object(PyTypeObject *type, int sizehint, SOCKET fd) self->epfd = fd; } if (self->epfd < 0) { - Py_DECREF(self); PyErr_SetFromErrno(PyExc_OSError); + Py_DECREF(self); return NULL; } @@ -1973,8 +1973,8 @@ newKqueue_Object(PyTypeObject *type, SOCKET fd) self->kqfd = fd; } if (self->kqfd < 0) { - Py_DECREF(self); PyErr_SetFromErrno(PyExc_OSError); + Py_DECREF(self); return NULL; } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index b4094b8c49a37f..8cc45e22549f30 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5398,8 +5398,8 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, } if (!SetHandleInformation((HANDLE)fd, HANDLE_FLAG_INHERIT, 0)) { - closesocket(fd); PyErr_SetFromWindowsErr(0); + closesocket(fd); return -1; } @@ -5613,8 +5613,9 @@ socket_gethostname(PyObject *self, PyObject *unused) name, &size)) { + PyErr_SetFromWindowsErr(0); PyMem_Free(name); - return PyErr_SetFromWindowsErr(0); + return NULL; } result = PyUnicode_FromWideChar(name, size); @@ -6212,8 +6213,8 @@ socket_dup(PyObject *self, PyObject *fdobj) } if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { - closesocket(newfd); PyErr_SetFromWindowsErr(0); + closesocket(newfd); return NULL; } #else @@ -6660,11 +6661,12 @@ socket_inet_ntop(PyObject *self, PyObject *args) /* inet_ntop guarantee NUL-termination of resulting string. */ retval = inet_ntop(af, packed_ip.buf, ip, sizeof(ip)); - PyBuffer_Release(&packed_ip); if (!retval) { PyErr_SetFromErrno(PyExc_OSError); + PyBuffer_Release(&packed_ip); return NULL; } else { + PyBuffer_Release(&packed_ip); return PyUnicode_FromString(retval); } } @@ -7003,8 +7005,8 @@ socket_if_nameindex(PyObject *self, PyObject *arg) ni = if_nameindex(); if (ni == NULL) { - Py_DECREF(list); PyErr_SetFromErrno(PyExc_OSError); + Py_DECREF(list); return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2f6f41367d3888..045b48c526a958 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -7101,7 +7101,7 @@ decode_code_page_errors(UINT code_page, if (err != ERROR_NO_UNICODE_TRANSLATION && err != ERROR_INSUFFICIENT_BUFFER) { - PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(err); goto error; } insize++; diff --git a/Python/fileutils.c b/Python/fileutils.c index 9c4998397c4ac8..9bc1de2db84006 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1790,6 +1790,7 @@ _Py_fopen_obj(PyObject *path, const char *mode) Py_END_ALLOW_THREADS } while (f == NULL && errno == EINTR && !(async_err = PyErr_CheckSignals())); + int saved_errno = errno; PyMem_Free(wpath); #else PyObject *bytes; @@ -1812,13 +1813,14 @@ _Py_fopen_obj(PyObject *path, const char *mode) Py_END_ALLOW_THREADS } while (f == NULL && errno == EINTR && !(async_err = PyErr_CheckSignals())); - + int saved_errno = errno; Py_DECREF(bytes); #endif if (async_err) return NULL; if (f == NULL) { + errno = saved_errno; PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path); return NULL; } From 482fad7f01567447b7259ebf58d62999fcdc5964 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 26 Aug 2023 17:13:33 -0700 Subject: [PATCH 347/598] gh-108295: Fix crashes with TypeVar weakrefs (#108517) --- Lib/test/test_typing.py | 10 ++++++++++ .../2023-08-26-08-38-57.gh-issue-108295.Pn0QRM.rst | 1 + Objects/typevarobject.c | 3 +++ 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-26-08-38-57.gh-issue-108295.Pn0QRM.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 38baf9546f8b0f..6447ed6078e178 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -544,6 +544,16 @@ def test_bad_var_substitution(self): with self.assertRaises(TypeError): list[T][arg] + def test_many_weakrefs(self): + # gh-108295: this used to segfault + for cls in (ParamSpec, TypeVarTuple, TypeVar): + with self.subTest(cls=cls): + vals = weakref.WeakValueDictionary() + + for x in range(100000): + vals[x] = cls(str(x)) + del vals + def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: """Renders templates with possible combinations of replacements. diff --git a/Misc/NEWS.d/next/Library/2023-08-26-08-38-57.gh-issue-108295.Pn0QRM.rst b/Misc/NEWS.d/next/Library/2023-08-26-08-38-57.gh-issue-108295.Pn0QRM.rst new file mode 100644 index 00000000000000..7e61ed4de75ce6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-26-08-38-57.gh-issue-108295.Pn0QRM.rst @@ -0,0 +1 @@ +Fix crashes related to use of weakrefs on :data:`typing.TypeVar`. diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 66122e36283ad0..ef1803d0fd5526 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -201,6 +201,7 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); _PyObject_ClearManagedDict(self); + PyObject_ClearWeakRefs(self); Py_TYPE(self)->tp_free(self); Py_DECREF(tp); @@ -743,6 +744,7 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); _PyObject_ClearManagedDict(self); + PyObject_ClearWeakRefs(self); Py_TYPE(self)->tp_free(self); Py_DECREF(tp); @@ -1022,6 +1024,7 @@ typevartuple_dealloc(PyObject *self) Py_DECREF(tvt->name); _PyObject_ClearManagedDict(self); + PyObject_ClearWeakRefs(self); Py_TYPE(self)->tp_free(self); Py_DECREF(tp); From 6cb48f049501c9f8c5be107e52f8eb359b5ac533 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 27 Aug 2023 12:31:29 +0900 Subject: [PATCH 348/598] gh-107265: Fix initialize/remove_tools for ENTER_EXECUTOR case (gh-108482) --- Python/instrumentation.c | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8c7a3a06c9b938..36459687be7936 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -566,7 +566,13 @@ de_instrument(PyCodeObject *code, int i, int event) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; - assert(opcode != ENTER_EXECUTOR); + if (opcode == ENTER_EXECUTOR) { + int oparg = instr->op.arg; + _PyExecutorObject *exec = code->co_executors->executors[oparg]; + opcode_ptr = &exec->vm_data.opcode; + opcode = *opcode_ptr; + assert(opcode != ENTER_EXECUTOR); + } if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; @@ -711,7 +717,22 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); - assert(opcode_has_event(_Py_GetBaseOpcode(code, offset))); + #ifndef NDEBUG + _Py_CODEUNIT co_instr = _PyCode_CODE(code)[offset]; + uint8_t opcode = co_instr.op.code; + uint8_t oparg = co_instr.op.arg; + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[oparg]; + assert(exec->vm_data.opcode != ENTER_EXECUTOR); + opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; + opcode = exec->vm_data.oparg; + } + else { + opcode = _Py_GetBaseOpcode(code, offset); + } + assert(opcode != ENTER_EXECUTOR); + assert(opcode_has_event(opcode)); + #endif _PyCoMonitoringData *monitoring = code->_co_monitoring; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; @@ -1282,9 +1303,16 @@ initialize_tools(PyCodeObject *code) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; - if (opcode == INSTRUMENTED_LINE) { + int oparg = instr->op.arg; + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[oparg]; + opcode = exec->vm_data.opcode; + oparg = exec->vm_data.oparg; + } + else if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[i].original_opcode; } + assert(opcode != ENTER_EXECUTOR); bool instrumented = is_instrumented(opcode); if (instrumented) { opcode = DE_INSTRUMENT[opcode]; @@ -1295,7 +1323,7 @@ initialize_tools(PyCodeObject *code) if (instrumented) { int8_t event; if (opcode == RESUME) { - event = instr->op.arg != 0; + event = oparg != 0; } else { event = EVENT_FOR_OPCODE[opcode]; From 1ac64237e6ce965064451ed57ae37271aeb9fbd3 Mon Sep 17 00:00:00 2001 From: qqwqqw689 <114795525+qqwqqw689@users.noreply.github.com> Date: Sun, 27 Aug 2023 13:19:49 +0800 Subject: [PATCH 349/598] gh-107453: Document errno.{ECANCELED,EOWNERDEAD,ENOTRECOVERABLE,ENOTSUP} (#107486) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/errno.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 5122c69697ef91..283e8b013265d9 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -511,6 +511,13 @@ defined by the module. The specific list of defined symbols is available as Operation not supported on transport endpoint +.. data:: ENOTSUP + + Operation not supported + + .. versionadded:: 3.2 + + .. data:: EPFNOSUPPORT Protocol family not supported @@ -666,3 +673,24 @@ defined by the module. The specific list of defined symbols is available as .. availability:: WASI, FreeBSD .. versionadded:: 3.11.1 + + +.. data:: ECANCELED + + Operation canceled + + .. versionadded:: 3.2 + + +.. data:: EOWNERDEAD + + Owner died + + .. versionadded:: 3.2 + + +.. data:: ENOTRECOVERABLE + + State not recoverable + + .. versionadded:: 3.2 From 7096a2be33619dc02c06a6dc30aac414a9eba462 Mon Sep 17 00:00:00 2001 From: R Date: Sun, 27 Aug 2023 15:22:27 +0800 Subject: [PATCH 350/598] gh-105052:update timeit function's description (#105060) --------- Co-authored-by: Terry Jan Reedy --- Doc/library/timeit.rst | 6 ++++-- Lib/timeit.py | 18 ++++++++++++------ ...3-05-29-14-10-24.gh-issue-105052.MGFwbm.rst | 1 + 3 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-05-29-14-10-24.gh-issue-105052.MGFwbm.rst diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index a559e0a2eb3dad..b3d2a1b9e0600f 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -86,9 +86,11 @@ The module defines three convenience functions and a public class: .. versionchanged:: 3.7 Default value of *repeat* changed from 3 to 5. + .. function:: default_timer() - The default timer, which is always :func:`time.perf_counter`. + The default timer, which is always time.perf_counter(), returns float seconds. + An alternative, time.perf_counter_ns, returns integer nanoseconds. .. versionchanged:: 3.3 :func:`time.perf_counter` is now the default timer. @@ -124,7 +126,7 @@ The module defines three convenience functions and a public class: Time *number* executions of the main statement. This executes the setup statement once, and then returns the time it takes to execute the main - statement a number of times, measured in seconds as a float. + statement a number of times. The default timer returns seconds as a float. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. diff --git a/Lib/timeit.py b/Lib/timeit.py index 0cf8db67723a49..02cfafaf36e5d1 100755 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -50,9 +50,9 @@ """ import gc +import itertools import sys import time -import itertools __all__ = ["Timer", "timeit", "repeat", "default_timer"] @@ -77,9 +77,11 @@ def inner(_it, _timer{init}): return _t1 - _t0 """ + def reindent(src, indent): """Helper to reindent a multi-line statement.""" - return src.replace("\n", "\n" + " "*indent) + return src.replace("\n", "\n" + " " * indent) + class Timer: """Class for timing execution speed of small code snippets. @@ -166,7 +168,7 @@ def timeit(self, number=default_number): To be precise, this executes the setup statement once, and then returns the time it takes to execute the main statement - a number of times, as a float measured in seconds. The + a number of times, as float seconds if using the default timer. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. @@ -228,16 +230,19 @@ def autorange(self, callback=None): return (number, time_taken) i *= 10 + def timeit(stmt="pass", setup="pass", timer=default_timer, number=default_number, globals=None): """Convenience function to create Timer object and call timeit method.""" return Timer(stmt, setup, timer, globals).timeit(number) + def repeat(stmt="pass", setup="pass", timer=default_timer, repeat=default_repeat, number=default_number, globals=None): """Convenience function to create Timer object and call repeat method.""" return Timer(stmt, setup, timer, globals).repeat(repeat, number) + def main(args=None, *, _wrap_timer=None): """Main program, used when run as a script. @@ -269,7 +274,7 @@ def main(args=None, *, _wrap_timer=None): timer = default_timer stmt = "\n".join(args) or "pass" - number = 0 # auto-determine + number = 0 # auto-determine setup = [] repeat = default_repeat verbose = 0 @@ -286,7 +291,7 @@ def main(args=None, *, _wrap_timer=None): time_unit = a else: print("Unrecognized unit. Please select nsec, usec, msec, or sec.", - file=sys.stderr) + file=sys.stderr) return 2 if o in ("-r", "--repeat"): repeat = int(a) @@ -320,7 +325,7 @@ def callback(number, time_taken): msg = "{num} loop{s} -> {secs:.{prec}g} secs" plural = (number != 1) print(msg.format(num=number, s='s' if plural else '', - secs=time_taken, prec=precision)) + secs=time_taken, prec=precision)) try: number, _ = t.autorange(callback) except: @@ -371,5 +376,6 @@ def format_time(dt): UserWarning, '', 0) return None + if __name__ == "__main__": sys.exit(main()) diff --git a/Misc/NEWS.d/next/Documentation/2023-05-29-14-10-24.gh-issue-105052.MGFwbm.rst b/Misc/NEWS.d/next/Documentation/2023-05-29-14-10-24.gh-issue-105052.MGFwbm.rst new file mode 100644 index 00000000000000..8fdc38d439f54f --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-05-29-14-10-24.gh-issue-105052.MGFwbm.rst @@ -0,0 +1 @@ +Update ``timeit`` doc to specify that time in seconds is just the default. From cd0a8aece974330ef44ffe4e0f2e8aa632e98438 Mon Sep 17 00:00:00 2001 From: Matthew James Kraai Date: Sun, 27 Aug 2023 04:26:23 -0700 Subject: [PATCH 351/598] Fix grammatical error in stringprep documentation (#108414) Remove the word "them", which didn't make grammatical sense. Co-authored-by: KRAAI, MATTHEW [VISUS] --- Doc/library/stringprep.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 5cfb533d802db4..c6d78a356d97bc 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -27,7 +27,7 @@ procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these -tables would be very large to represent them as dictionaries or lists, the +tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. From 09343dba44cdb5c279ec51df34552ef451434958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 27 Aug 2023 15:09:40 +0200 Subject: [PATCH 352/598] Clarify distinction between datetime module and class in deprecation messages (GH-108073) --- Modules/_datetimemodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 454be6efc09d35..191db3f84088d5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5136,9 +5136,9 @@ static PyObject * datetime_utcnow(PyObject *cls, PyObject *dummy) { if (PyErr_WarnEx(PyExc_DeprecationWarning, - "datetime.utcnow() is deprecated and scheduled for removal in a " + "datetime.datetime.utcnow() is deprecated and scheduled for removal in a " "future version. Use timezone-aware objects to represent datetimes " - "in UTC: datetime.now(datetime.UTC).", 1)) + "in UTC: datetime.datetime.now(datetime.UTC).", 1)) { return NULL; } @@ -5179,9 +5179,9 @@ static PyObject * datetime_utcfromtimestamp(PyObject *cls, PyObject *args) { if (PyErr_WarnEx(PyExc_DeprecationWarning, - "datetime.utcfromtimestamp() is deprecated and scheduled for removal " + "datetime.datetime.utcfromtimestamp() is deprecated and scheduled for removal " "in a future version. Use timezone-aware objects to represent " - "datetimes in UTC: datetime.fromtimestamp(timestamp, datetime.UTC).", 1)) + "datetimes in UTC: datetime.datetime.fromtimestamp(timestamp, datetime.UTC).", 1)) { return NULL; } From 042aa88bcc6541cb8b312f1119452f7a58a5b4df Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 27 Aug 2023 08:59:40 -0500 Subject: [PATCH 353/598] gh-108322: Optimize statistics.NormalDist.samples() (gh-108324) --- Doc/library/statistics.rst | 5 +++++ Lib/statistics.py | 12 +++++++----- .../2023-08-22-12-05-47.gh-issue-108322.kf3NJX.rst | 2 ++ 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-22-12-05-47.gh-issue-108322.kf3NJX.rst diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 483ebea67f0c6d..368b2a17cef997 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -828,6 +828,11 @@ of applications in statistics. number generator. This is useful for creating reproducible results, even in a multi-threading context. + .. versionchanged:: 3.13 + + Switched to a faster algorithm. To reproduce samples from previous + versions, use :func:`random.seed` and :func:`random.gauss`. + .. method:: NormalDist.pdf(x) Using a `probability density function (pdf) diff --git a/Lib/statistics.py b/Lib/statistics.py index a8036e9928c464..96c803483057e7 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1135,7 +1135,7 @@ def linear_regression(x, y, /, *, proportional=False): >>> noise = NormalDist().samples(5, seed=42) >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)] >>> linear_regression(x, y) #doctest: +ELLIPSIS - LinearRegression(slope=3.09078914170..., intercept=1.75684970486...) + LinearRegression(slope=3.17495..., intercept=1.00925...) If *proportional* is true, the independent variable *x* and the dependent variable *y* are assumed to be directly proportional. @@ -1148,7 +1148,7 @@ def linear_regression(x, y, /, *, proportional=False): >>> y = [3 * x[i] + noise[i] for i in range(5)] >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS - LinearRegression(slope=3.02447542484..., intercept=0.0) + LinearRegression(slope=2.90475..., intercept=0.0) """ n = len(x) @@ -1279,9 +1279,11 @@ def from_samples(cls, data): def samples(self, n, *, seed=None): "Generate *n* samples for a given mean and standard deviation." - gauss = random.gauss if seed is None else random.Random(seed).gauss - mu, sigma = self._mu, self._sigma - return [gauss(mu, sigma) for _ in repeat(None, n)] + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] def pdf(self, x): "Probability density function. P(x <= X < x+dx) / dx" diff --git a/Misc/NEWS.d/next/Library/2023-08-22-12-05-47.gh-issue-108322.kf3NJX.rst b/Misc/NEWS.d/next/Library/2023-08-22-12-05-47.gh-issue-108322.kf3NJX.rst new file mode 100644 index 00000000000000..5416c01a43f113 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-22-12-05-47.gh-issue-108322.kf3NJX.rst @@ -0,0 +1,2 @@ +Speed-up NormalDist.samples() by using the inverse CDF method instead of +calling random.gauss(). From a429eafef2d86eafc007ac19682e7d372c32da31 Mon Sep 17 00:00:00 2001 From: Ori Avtalion Date: Sun, 27 Aug 2023 22:50:59 +0300 Subject: [PATCH 354/598] gh-108542: Fix incorrect module name in NEWS entry for gh-105475 (#108543) --- .../next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst b/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst index 3030477c8245b5..f47754e172251f 100644 --- a/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst +++ b/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst @@ -1 +1 @@ -Fix bugs in :mod:`pickle` where exceptions could be overwritten. +Fix bugs in :mod:`errno` where exceptions could be overwritten. From 38afa4af9bfc8297a5ee270c37f3f120a04297ea Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 27 Aug 2023 22:35:27 +0200 Subject: [PATCH 355/598] gh-107801: Document io.TextIOWrapper.tell (#108265) --- Doc/library/io.rst | 10 +++++++++- Modules/_io/clinic/textio.c.h | 8 ++++++-- Modules/_io/textio.c | 7 ++++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 792bf43d9811bb..25f4d0d5841333 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1054,13 +1054,21 @@ Text I/O * ``seek(0, SEEK_SET)``: Rewind to the start of the stream. * ``seek(cookie, SEEK_SET)``: Restore a previous position; - *cookie* **must be** a number returned by :meth:`!tell`. + *cookie* **must be** a number returned by :meth:`tell`. * ``seek(0, SEEK_END)``: Fast-forward to the end of the stream. * ``seek(0, SEEK_CUR)``: Leave the current stream position unchanged. Any other argument combinations are invalid, and may raise exceptions. + .. method:: tell() + + Return the stream position as an opaque number. + The return value of :meth:`!tell` can be given as input to :meth:`seek`, + to restore a previous stream position. + + + .. class:: StringIO(initial_value='', newline='\n') A text stream using an in-memory text buffer. It inherits diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 03d5facedfb513..ce6efbe20b6bba 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -813,7 +813,11 @@ _io_TextIOWrapper_seek(textio *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(_io_TextIOWrapper_tell__doc__, "tell($self, /)\n" "--\n" -"\n"); +"\n" +"Return the stream position as an opaque number.\n" +"\n" +"The return value of tell() can be given as input to seek(), to restore a\n" +"previous stream position."); #define _IO_TEXTIOWRAPPER_TELL_METHODDEF \ {"tell", (PyCFunction)_io_TextIOWrapper_tell, METH_NOARGS, _io_TextIOWrapper_tell__doc__}, @@ -976,4 +980,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=c3a8eb2591be1bf7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=390af0e65a0d02c0 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index d87bd0054d4438..8e44d453269a71 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2640,11 +2640,16 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) /*[clinic input] _io.TextIOWrapper.tell + +Return the stream position as an opaque number. + +The return value of tell() can be given as input to seek(), to restore a +previous stream position. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_tell_impl(textio *self) -/*[clinic end generated code: output=4f168c08bf34ad5f input=9a2caf88c24f9ddf]*/ +/*[clinic end generated code: output=4f168c08bf34ad5f input=0852d627d76fb520]*/ { PyObject *res; PyObject *posobj = NULL; From fecb9faf0b2df6a219696502a34b918c5d2bfe9d Mon Sep 17 00:00:00 2001 From: John Micco Date: Sun, 27 Aug 2023 14:07:09 -0700 Subject: [PATCH 356/598] gh-108465: Use compiler basename when determining compiler flags (#108392) Note: GNU Autoconf discourages the use of 'basename', and recommends 'expr' instead. Co-authored-by: Erlend E. Aasland --- configure | 9 ++++++--- configure.ac | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/configure b/configure index 34e874c49cce0a..15ca703bb9163f 100755 --- a/configure +++ b/configure @@ -8562,7 +8562,10 @@ printf "%s\n" "$as_me: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&6;} fi fi LLVM_PROF_ERR=no -case $CC in + +# GNU Autoconf recommends the use of expr instead of basename. +CC_BASENAME=$(expr "//$CC" : '.*/\(.*\)') +case "$CC_BASENAME" in *clang*) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" @@ -9391,7 +9394,7 @@ fi # ICC doesn't recognize the option, but only emits a warning ## XXX does it emit an unused result warning and can it be disabled? - case $CC in #( + case "$CC_BASENAME" in #( *icc*) : ac_cv_disable_unused_result_warning=no @@ -9972,7 +9975,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac -case "$CC" in +case "$CC_BASENAME" in *mpicc*) CFLAGS_NODIST="$CFLAGS_NODIST" ;; diff --git a/configure.ac b/configure.ac index 4b343b5d531c4a..fab394ec471ce7 100644 --- a/configure.ac +++ b/configure.ac @@ -1794,7 +1794,10 @@ then fi fi LLVM_PROF_ERR=no -case $CC in + +# GNU Autoconf recommends the use of expr instead of basename. +AS_VAR_SET([CC_BASENAME], [$(expr "//$CC" : '.*/\(.*\)')]) +case "$CC_BASENAME" in *clang*) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" @@ -2225,7 +2228,7 @@ yes) # ICC doesn't recognize the option, but only emits a warning ## XXX does it emit an unused result warning and can it be disabled? - AS_CASE([$CC], + AS_CASE(["$CC_BASENAME"], [*icc*], [ac_cv_disable_unused_result_warning=no] [PY_CHECK_CC_WARNING([disable], [unused-result])]) AS_VAR_IF([ac_cv_disable_unused_result_warning], [yes], @@ -2471,7 +2474,7 @@ yes) ;; esac -case "$CC" in +case "$CC_BASENAME" in *mpicc*) CFLAGS_NODIST="$CFLAGS_NODIST" ;; From d0160c7c22c8dff0a61c49b5304244df6e36465e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 28 Aug 2023 00:18:32 +0200 Subject: [PATCH 357/598] gh-64662: Add virtual table support to sqlite3.Connection.iterdump (#108340) Co-authored-by: Aviv Palivoda --- Doc/whatsnew/3.13.rst | 3 +++ Lib/sqlite3/dump.py | 20 ++++++++++++------- Lib/test/test_sqlite3/test_dump.py | 20 +++++++++++++++++++ ...3-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst | 2 ++ 4 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 4ff12b16d00266..2770207c7097a5 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -165,6 +165,9 @@ sqlite3 object is not :meth:`closed ` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) +* Add support for virtual tables to :meth:`sqlite3.Connection.iterdump`. + (Contributed by Aviv Palivoda in :gh:`64662`.) + tkinter ------- diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index cf73cc33810cb1..ead3360ce67608 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -24,6 +24,7 @@ def _iterdump(connection): directly but instead called from the Connection method, iterdump(). """ + writeable_schema = False cu = connection.cursor() yield('BEGIN TRANSACTION;') @@ -50,13 +51,15 @@ def _iterdump(connection): yield('ANALYZE "sqlite_master";') elif table_name.startswith('sqlite_'): continue - # NOTE: Virtual table support not implemented - #elif sql.startswith('CREATE VIRTUAL TABLE'): - # qtable = table_name.replace("'", "''") - # yield("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)"\ - # "VALUES('table','{0}','{0}',0,'{1}');".format( - # qtable, - # sql.replace("''"))) + elif sql.startswith('CREATE VIRTUAL TABLE'): + if not writeable_schema: + writeable_schema = True + yield('PRAGMA writable_schema=ON;') + yield("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table',{0},{0},0,{1});".format( + _quote_value(table_name), + _quote_value(sql), + )) else: yield('{0};'.format(sql)) @@ -85,6 +88,9 @@ def _iterdump(connection): for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) + if writeable_schema: + yield('PRAGMA writable_schema=OFF;') + # gh-79009: Yield statements concerning the sqlite_sequence table at the # end of the transaction. for row in sqlite_sequence: diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 5f6811fb5cc0a5..3107e1b165d950 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -113,6 +113,26 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + def test_dump_virtual_tables(self): + # gh-64662 + expected = [ + "BEGIN TRANSACTION;", + "PRAGMA writable_schema=ON;", + ("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','test','test',0,'CREATE VIRTUAL TABLE test USING fts4(example)');"), + "CREATE TABLE 'test_content'(docid INTEGER PRIMARY KEY, 'c0example');", + "CREATE TABLE 'test_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + ("CREATE TABLE 'test_segdir'(level INTEGER,idx INTEGER,start_block INTEGER," + "leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));"), + "CREATE TABLE 'test_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + "CREATE TABLE 'test_stat'(id INTEGER PRIMARY KEY, value BLOB);", + "PRAGMA writable_schema=OFF;", + "COMMIT;" + ] + self.cu.execute("CREATE VIRTUAL TABLE test USING fts4(example)") + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst b/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst new file mode 100644 index 00000000000000..3fddc8c43aa1db --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst @@ -0,0 +1,2 @@ +Add support for virtual tables to :meth:`sqlite3.Connection.iterdump`. Patch +by Aviv Palivoda. From 5d936b64796261373429c86cdf90b1d8d8acefba Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 28 Aug 2023 06:19:31 +0100 Subject: [PATCH 358/598] GH-108202: Combine documentation of ``calendar`` constants (#108492) --- Doc/library/calendar.rst | 109 ++++++++++++++++++--------------------- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.13.rst | 4 +- 3 files changed, 52 insertions(+), 63 deletions(-) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 3c097f4cf7abd9..1868eb1cd359c4 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -28,58 +28,6 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is 2 BC, and so on. -.. class:: Day - - Enumeration defining the days of the week as integer constants, from 0 to 6. - - .. attribute:: MONDAY - - .. attribute:: TUESDAY - - .. attribute:: WEDNESDAY - - .. attribute:: THURSDAY - - .. attribute:: FRIDAY - - .. attribute:: SATURDAY - - .. attribute:: SUNDAY - - .. versionadded:: 3.12 - - -.. class:: Month - - Enumeration defining months of the year as integer constants, from 1 to 12. - - .. attribute:: JANUARY - - .. attribute:: FEBRUARY - - .. attribute:: MARCH - - .. attribute:: APRIL - - .. attribute:: MAY - - .. attribute:: JUNE - - .. attribute:: JULY - - .. attribute:: AUGUST - - .. attribute:: SEPTEMBER - - .. attribute:: OCTOBER - - .. attribute:: NOVEMBER - - .. attribute:: DECEMBER - - .. versionadded:: 3.12 - - .. class:: Calendar(firstweekday=0) Creates a :class:`Calendar` object. *firstweekday* is an integer specifying the @@ -446,6 +394,29 @@ The :mod:`calendar` module exports the following data attributes: An array that represents the abbreviated days of the week in the current locale. +.. data:: MONDAY + TUESDAY + WEDNESDAY + THURSDAY + FRIDAY + SATURDAY + SUNDAY + + Aliases for the days of the week, + where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. + + .. versionadded:: 3.12 + + +.. class:: Day + + Enumeration defining days of the week as integer constants. + The members of this enumeration are exported to the module scope as + :data:`MONDAY` through :data:`SUNDAY`. + + .. versionadded:: 3.12 + + .. data:: month_name An array that represents the months of the year in the current locale. This @@ -459,15 +430,33 @@ The :mod:`calendar` module exports the following data attributes: locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. -.. data:: MONDAY - TUESDAY - WEDNESDAY - THURSDAY - FRIDAY - SATURDAY - SUNDAY - Aliases for day numbers, where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. +.. data:: JANUARY + FEBRUARY + MARCH + APRIL + MAY + JUNE + JULY + AUGUST + SEPTEMBER + OCTOBER + NOVEMBER + DECEMBER + + Aliases for the months of the year, + where ``JANUARY`` is ``1`` and ``DECEMBER`` is ``12``. + + .. versionadded:: 3.12 + + +.. class:: Month + + Enumeration defining months of the year as integer constants. + The members of this enumeration are exported to the module scope as + :data:`JANUARY` through :data:`DECEMBER`. + + .. versionadded:: 3.12 The :mod:`calendar` module defines the following exceptions: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index b557e6cde6a209..c5a5f7e6bd0235 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1013,7 +1013,7 @@ Deprecated (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) * :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are deprecated and - replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. + replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) * :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 2770207c7097a5..499155ae23f887 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -438,8 +438,8 @@ although there is currently no date scheduled for their removal. * Delegation of ``int()`` to ``__trunc__()`` method. * :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are - deprecated and replaced by :data:`calendar.Month.JANUARY` and - :data:`calendar.Month.FEBRUARY`. + deprecated and replaced by :data:`calendar.JANUARY` and + :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) * :mod:`datetime`: From 72b615ab015ccff8a92e22c5b5f97fa8aca3ba1f Mon Sep 17 00:00:00 2001 From: nikkie Date: Mon, 28 Aug 2023 20:19:29 +0900 Subject: [PATCH 359/598] Fix typo in typing docs: Remove redundant backtick (#108559) --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index fad945ffc8210a..18e15f304f9f80 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -462,7 +462,7 @@ contrast, a variable annotated with ``type[C]`` (or themselves -- specifically, it will accept the *class object* of ``C``. For example:: - a = 3 # Has type ``int``` + a = 3 # Has type ``int`` b = int # Has type ``type[int]`` c = type(a) # Also has type ``type[int]`` From 8db451ceb17cf8ec92c8aed2e0a41c78d07f9f45 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 28 Aug 2023 14:02:26 +0200 Subject: [PATCH 360/598] gh-64662: Fix NEWS entry and remove What's New entry (#108565) --- Doc/whatsnew/3.13.rst | 3 --- .../next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 499155ae23f887..d31d458b2fb9c9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -165,9 +165,6 @@ sqlite3 object is not :meth:`closed ` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) -* Add support for virtual tables to :meth:`sqlite3.Connection.iterdump`. - (Contributed by Aviv Palivoda in :gh:`64662`.) - tkinter ------- diff --git a/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst b/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst index 3fddc8c43aa1db..1b4c33a82b0b86 100644 --- a/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst +++ b/Misc/NEWS.d/next/Library/2023-08-22-22-29-42.gh-issue-64662.jHl_Bt.rst @@ -1,2 +1,2 @@ -Add support for virtual tables to :meth:`sqlite3.Connection.iterdump`. Patch +Fix support for virtual tables in :meth:`sqlite3.Connection.iterdump`. Patch by Aviv Palivoda. From 0e8b3fc718c8a1c4de558c553d9e05049c1dbec6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 28 Aug 2023 14:17:34 +0200 Subject: [PATCH 361/598] gh-108550: Speed up sqlite3 tests (#108551) Refactor the CLI so we can easily invoke it and mock command-line arguments. Adapt the CLI tests so we no longer have to launch a separate process. Disable the busy handler for all concurrency tests; we have full control over the order of the SQLite C API calls, so we can safely do this. The sqlite3 test suite now completes ~8 times faster than before. Co-authored-by: Serhiy Storchaka --- Lib/sqlite3/__main__.py | 9 +- Lib/test/test_sqlite3/test_cli.py | 148 +++++++++------------ Lib/test/test_sqlite3/test_dbapi.py | 2 +- Lib/test/test_sqlite3/test_transactions.py | 16 +-- 4 files changed, 74 insertions(+), 101 deletions(-) diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 3228dbc09d502a..10a2e9e0a202a4 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -62,7 +62,7 @@ def runsource(self, source, filename="", symbol="single"): return False -def main(): +def main(*args): parser = ArgumentParser( description="Python sqlite3 CLI", prog="python -m sqlite3", @@ -86,7 +86,7 @@ def main(): version=f"SQLite version {sqlite3.sqlite_version}", help="Print underlying SQLite library version", ) - args = parser.parse_args() + args = parser.parse_args(*args) if args.filename == ":memory:": db_name = "a transient in-memory database" @@ -120,5 +120,8 @@ def main(): finally: con.close() + sys.exit(0) -main() + +if __name__ == "__main__": + main(sys.argv) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index d374f8ee4fc8d3..e681f5c976b7b4 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,42 +1,35 @@ """sqlite3 CLI tests.""" - -import sqlite3 as sqlite -import subprocess -import sys +import sqlite3 import unittest -from test.support import SHORT_TIMEOUT, requires_subprocess +from sqlite3.__main__ import main as cli from test.support.os_helper import TESTFN, unlink +from test.support import captured_stdout, captured_stderr, captured_stdin -@requires_subprocess() class CommandLineInterface(unittest.TestCase): def _do_test(self, *args, expect_success=True): - with subprocess.Popen( - [sys.executable, "-Xutf8", "-m", "sqlite3", *args], - encoding="utf-8", - bufsize=0, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as proc: - proc.wait() - if expect_success == bool(proc.returncode): - self.fail("".join(proc.stderr)) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - if expect_success: - self.assertEqual(stderr, "") - else: - self.assertEqual(stdout, "") - return stdout, stderr + with ( + captured_stdout() as out, + captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + cli(args) + return out.getvalue(), err.getvalue(), cm.exception.code def expect_success(self, *args): - out, _ = self._do_test(*args) + out, err, code = self._do_test(*args) + self.assertEqual(code, 0, + "\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(err, "") return out def expect_failure(self, *args): - _, err = self._do_test(*args, expect_success=False) + out, err, code = self._do_test(*args, expect_success=False) + self.assertNotEqual(code, 0, + "\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(out, "") return err def test_cli_help(self): @@ -45,7 +38,7 @@ def test_cli_help(self): def test_cli_version(self): out = self.expect_success("-v") - self.assertIn(sqlite.sqlite_version, out) + self.assertIn(sqlite3.sqlite_version, out) def test_cli_execute_sql(self): out = self.expect_success(":memory:", "select 1") @@ -68,87 +61,68 @@ def test_cli_on_disk_db(self): self.assertIn("(0,)", out) -@requires_subprocess() class InteractiveSession(unittest.TestCase): - TIMEOUT = SHORT_TIMEOUT / 10. MEMORY_DB_MSG = "Connected to a transient in-memory database" PS1 = "sqlite> " PS2 = "... " - def start_cli(self, *args): - return subprocess.Popen( - [sys.executable, "-Xutf8", "-m", "sqlite3", *args], - encoding="utf-8", - bufsize=0, - stdin=subprocess.PIPE, - # Note: the banner is printed to stderr, the prompt to stdout. - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - def expect_success(self, proc): - proc.wait() - if proc.returncode: - self.fail("".join(proc.stderr)) + def run_cli(self, *args, commands=()): + with ( + captured_stdin() as stdin, + captured_stdout() as stdout, + captured_stderr() as stderr, + self.assertRaises(SystemExit) as cm + ): + for cmd in commands: + stdin.write(cmd + "\n") + stdin.seek(0) + cli(args) + + out = stdout.getvalue() + err = stderr.getvalue() + self.assertEqual(cm.exception.code, 0, + f"Unexpected failure: {args=}\n{out}\n{err}") + return out, err def test_interact(self): - with self.start_cli() as proc: - out, err = proc.communicate(timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) + out, err = self.run_cli() + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(self.PS1, out) def test_interact_quit(self): - with self.start_cli() as proc: - out, err = proc.communicate(input=".quit", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) + out, err = self.run_cli(commands=(".quit",)) + self.assertIn(self.PS1, out) def test_interact_version(self): - with self.start_cli() as proc: - out, err = proc.communicate(input=".version", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(sqlite.sqlite_version, out) - self.expect_success(proc) + out, err = self.run_cli(commands=(".version",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(sqlite3.sqlite_version, out) def test_interact_valid_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="select 1;", - timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("(1,)", out) - self.expect_success(proc) + out, err = self.run_cli(commands=("SELECT 1;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn("(1,)", out) def test_interact_valid_multiline_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="select 1\n;", - timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS2, out) - self.assertIn("(1,)", out) - self.expect_success(proc) + out, err = self.run_cli(commands=("SELECT 1\n;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(self.PS2, out) + self.assertIn("(1,)", out) def test_interact_invalid_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="sel;", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("OperationalError (SQLITE_ERROR)", err) - self.expect_success(proc) + out, err = self.run_cli(commands=("sel;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn("OperationalError (SQLITE_ERROR)", err) def test_interact_on_disk_file(self): self.addCleanup(unlink, TESTFN) - with self.start_cli(TESTFN) as proc: - out, err = proc.communicate(input="create table t(t);", - timeout=self.TIMEOUT) - self.assertIn(TESTFN, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) - with self.start_cli(TESTFN, "select count(t) from t") as proc: - out = proc.stdout.read() - err = proc.stderr.read() - self.assertIn("(0,)", out) - self.expect_success(proc) + + out, err = self.run_cli(TESTFN, commands=("CREATE TABLE t(t);",)) + self.assertIn(TESTFN, err) + self.assertIn(self.PS1, out) + + out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) + self.assertIn("(0,)", out) if __name__ == "__main__": diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 9dedbdbc4bb6d2..f3efe0f52f4fd7 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1854,7 +1854,7 @@ def test_on_conflict_replace(self): @requires_subprocess() class MultiprocessTests(unittest.TestCase): - CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000. # Defaults to 30 ms + CONNECTION_TIMEOUT = 0 # Disable the busy timeout. def tearDown(self): unlink(TESTFN) diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py index b7b231d2225852..a3de7a7a82ec1c 100644 --- a/Lib/test/test_sqlite3/test_transactions.py +++ b/Lib/test/test_sqlite3/test_transactions.py @@ -24,7 +24,6 @@ import sqlite3 as sqlite from contextlib import contextmanager -from test.support import LOOPBACK_TIMEOUT from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok @@ -32,15 +31,14 @@ from .util import MemoryDatabaseMixin -TIMEOUT = LOOPBACK_TIMEOUT / 10 - - class TransactionTests(unittest.TestCase): def setUp(self): - self.con1 = sqlite.connect(TESTFN, timeout=TIMEOUT) + # We can disable the busy handlers, since we control + # the order of SQLite C API operations. + self.con1 = sqlite.connect(TESTFN, timeout=0) self.cur1 = self.con1.cursor() - self.con2 = sqlite.connect(TESTFN, timeout=TIMEOUT) + self.con2 = sqlite.connect(TESTFN, timeout=0) self.cur2 = self.con2.cursor() def tearDown(self): @@ -120,10 +118,8 @@ def test_raise_timeout(self): self.cur2.execute("insert into test(i) values (5)") def test_locking(self): - """ - This tests the improved concurrency with pysqlite 2.3.4. You needed - to roll back con2 before you could commit con1. - """ + # This tests the improved concurrency with pysqlite 2.3.4. You needed + # to roll back con2 before you could commit con1. self.cur1.execute("create table test(i)") self.cur1.execute("insert into test(i) values (5)") with self.assertRaises(sqlite.OperationalError): From d90973340bf5ac35e0b35e99239cd37c46a30910 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 28 Aug 2023 14:41:05 +0200 Subject: [PATCH 362/598] gh-104683: Argument Clinic: Refactor the module and class resolver (#108552) --- Lib/test/test_clinic.py | 4 ++-- Tools/clinic/clinic.py | 26 ++++++++++---------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index e7039d59070597..fcccf988d3a7dd 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2324,10 +2324,10 @@ def test_invalid_legacy_converter(self): self.expect_failure(block, err, lineno=1) def test_parent_class_or_module_does_not_exist(self): - err = "Parent class or module 'z' does not exist" + err = "Parent class or module 'baz' does not exist" block = """ module m - z.func + baz.func """ self.expect_failure(block, err, lineno=1) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 70ec18f726e3f4..1fb53c3483123d 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2554,7 +2554,7 @@ def parse(self, input: str) -> str: return printer.f.getvalue() def _module_and_class( - self, fields: Iterable[str] + self, fields: Sequence[str] ) -> tuple[Module | Clinic, Class | None]: """ fields should be an iterable of field names. @@ -2563,26 +2563,20 @@ def _module_and_class( this function is only ever used to find the parent of where a new class/module should go. """ - parent: Clinic | Module | Class - child: Module | Class | None - module: Clinic | Module + parent: Clinic | Module | Class = self + module: Clinic | Module = self cls: Class | None = None - so_far: list[str] = [] - parent = module = self - - for field in fields: - so_far.append(field) + for idx, field in enumerate(fields): if not isinstance(parent, Class): - child = parent.modules.get(field) - if child: - parent = module = child + if field in parent.modules: + parent = module = parent.modules[field] continue - child = parent.classes.get(field) - if not child: - fullname = ".".join(so_far) + if field in parent.classes: + parent = cls = parent.classes[field] + else: + fullname = ".".join(fields[idx:]) fail(f"Parent class or module {fullname!r} does not exist.") - cls = parent = child return module, cls From bc5356bb5d7e3eda44128e89a695c05066e0840b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 28 Aug 2023 16:04:27 +0300 Subject: [PATCH 363/598] gh-108494: Argument Clinic: fix support of Limited C API (GH-108536) --- Lib/test/test_call.py | 11 +- Lib/test/test_enum.py | 6 +- Lib/test/test_pyexpat.py | 8 +- Modules/_localemodule.c | 1 + Modules/_sqlite/module.c | 2 +- Modules/_struct.c | 3 +- Modules/_tracemalloc.c | 1 + Modules/clinic/_testclinic_limited.c.h | 5 +- PC/winreg.c | 3 +- Tools/clinic/clinic.py | 221 ++++++++++++++----------- 10 files changed, 147 insertions(+), 114 deletions(-) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 008a8c1f0cb876..b1c78d7136fc9b 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -65,7 +65,8 @@ def test_varargs3(self): self.assertRaisesRegex(TypeError, msg, int.from_bytes, b'a', 'little', False) def test_varargs1min(self): - msg = r"get expected at least 1 argument, got 0" + msg = (r"get\(\) takes at least 1 argument \(0 given\)|" + r"get expected at least 1 argument, got 0") self.assertRaisesRegex(TypeError, msg, {}.get) msg = r"expected 1 argument, got 0" @@ -76,11 +77,13 @@ def test_varargs2min(self): self.assertRaisesRegex(TypeError, msg, getattr) def test_varargs1max(self): - msg = r"input expected at most 1 argument, got 2" + msg = (r"input\(\) takes at most 1 argument \(2 given\)|" + r"input expected at most 1 argument, got 2") self.assertRaisesRegex(TypeError, msg, input, 1, 2) def test_varargs2max(self): - msg = r"get expected at most 2 arguments, got 3" + msg = (r"get\(\) takes at most 2 arguments \(3 given\)|" + r"get expected at most 2 arguments, got 3") self.assertRaisesRegex(TypeError, msg, {}.get, 1, 2, 3) def test_varargs1_kw(self): @@ -96,7 +99,7 @@ def test_varargs3_kw(self): self.assertRaisesRegex(TypeError, msg, bool, x=2) def test_varargs4_kw(self): - msg = r"^list[.]index\(\) takes no keyword arguments$" + msg = r"^(list[.])?index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, [].index, x=2) def test_varargs5_kw(self): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 6e128439fa3569..36a1ee47640849 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2799,11 +2799,13 @@ class SecondFailedStrEnum(CustomStrEnum): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = 2 # this will become '2' - with self.assertRaisesRegex(TypeError, '.encoding. must be str, not '): + with self.assertRaisesRegex(TypeError, + r"argument (2|'encoding') must be str, not "): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = b'2', sys.getdefaultencoding - with self.assertRaisesRegex(TypeError, '.errors. must be str, not '): + with self.assertRaisesRegex(TypeError, + r"argument (3|'errors') must be str, not "): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = b'2', 'ascii', 9 diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 863c1194672c1c..abe1ad517d2246 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -281,12 +281,10 @@ def test_legal(self): expat.ParserCreate(namespace_separator=' ') def test_illegal(self): - try: + with self.assertRaisesRegex(TypeError, + r"ParserCreate\(\) argument (2|'namespace_separator') " + r"must be str or None, not int"): expat.ParserCreate(namespace_separator=42) - self.fail() - except TypeError as e: - self.assertEqual(str(e), - "ParserCreate() argument 'namespace_separator' must be str or None, not int") try: expat.ParserCreate(namespace_separator='too long') diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index e4b956b136dd1d..1847a4811e8ee9 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -11,6 +11,7 @@ This software comes with no warranty. Use at your own risk. #include "Python.h" #include "pycore_fileutils.h" +#include "pycore_pymem.h" // _PyMem_Strdup #include #include diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index dd45ffc1988f7e..46fed9f13281f3 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -244,7 +244,7 @@ load_functools_lru_cache(PyObject *module) static PyMethodDef module_methods[] = { PYSQLITE_ADAPT_METHODDEF PYSQLITE_COMPLETE_STATEMENT_METHODDEF - PYSQLITE_CONNECT_METHODDEF + {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, PYSQLITE_ENABLE_CALLBACK_TRACE_METHODDEF PYSQLITE_REGISTER_ADAPTER_METHODDEF PYSQLITE_REGISTER_CONVERTER_METHODDEF diff --git a/Modules/_struct.c b/Modules/_struct.c index 4da654231cdf00..4ae21cce74f609 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -108,6 +108,7 @@ class cache_struct_converter(CConverter): type = 'PyStructObject *' converter = 'cache_struct_converter' c_default = "NULL" + broken_limited_capi = True def parse_arg(self, argname, displayname): return """ @@ -120,7 +121,7 @@ class cache_struct_converter(CConverter): def cleanup(self): return "Py_XDECREF(%s);\n" % self.name [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=d6746621c2fb1a7d]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=14e83804f599ed8f]*/ static int cache_struct_converter(PyObject *, PyObject *, PyStructObject **); diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index f3f4af9aba08c1..6dba3cac01c1c8 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_tracemalloc.h" // _PyTraceMalloc_IsTracing #include "clinic/_tracemalloc.c.h" diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index 9b0032528ff746..93e7d7f98769cc 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -37,8 +37,7 @@ my_int_func(PyObject *module, PyObject *arg_) int arg; int _return_value; - arg = PyLong_AsInt(arg_); - if (arg == -1 && PyErr_Occurred()) { + if (!PyArg_Parse(arg_, "i:my_int_func", &arg)) { goto exit; } _return_value = my_int_func_impl(module, arg); @@ -82,4 +81,4 @@ my_int_sum(PyObject *module, PyObject *args) exit: return return_value; } -/*[clinic end generated code: output=f9f7209255bb969e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dcd5203d0d29df3a input=a9049054013a1b77]*/ diff --git a/PC/winreg.c b/PC/winreg.c index ed6258d2b33e56..ee47a3fd40fc86 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -220,6 +220,7 @@ class DWORD_converter(unsigned_long_converter): class HKEY_converter(CConverter): type = 'HKEY' converter = 'clinic_HKEY_converter' + broken_limited_capi = True def parse_arg(self, argname, displayname): return """ @@ -249,7 +250,7 @@ class self_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = (PyObject *)_return_value;\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=17e645060c7b8ae1]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=f8cb7034338aeaba]*/ #include "clinic/winreg.c.h" diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1fb53c3483123d..723592fa266c4c 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -975,6 +975,8 @@ def deprecate_keyword_use( func: Function, params: dict[int, Parameter], argname_fmt: str | None, + *, + limited_capi: bool, ) -> str: assert len(params) > 0 last_param = next(reversed(params.values())) @@ -986,7 +988,7 @@ def deprecate_keyword_use( if p.is_optional(): if argname_fmt: conditions.append(f"nargs < {i+1} && {argname_fmt % i}") - elif func.kind.new_or_init: + elif func.kind.new_or_init or limited_capi: conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") containscheck = "PyDict_Contains" else: @@ -998,7 +1000,9 @@ def deprecate_keyword_use( if len(conditions) > 1: condition = f"(({condition}))" if last_param.is_optional(): - if func.kind.new_or_init: + if limited_capi: + condition = f"kwargs && PyDict_Size(kwargs) && {condition}" + elif func.kind.new_or_init: condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" else: condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" @@ -1170,6 +1174,14 @@ def parser_body( add(field) return linear_format(output(), parser_declarations=declarations) + limited_capi = clinic.limited_capi + if limited_capi and (requires_defining_class or pseudo_args or + (any(p.is_optional() for p in parameters) and + any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or + any(c.broken_limited_capi for c in converters)): + warn(f"Function {f.full_name} cannot use limited C API") + limited_capi = False + parsearg: str | None if not parameters: parser_code: list[str] | None @@ -1230,8 +1242,11 @@ def parser_body( {c_basename}({self_type}{self_name}, PyObject *%s) """ % argname) - displayname = parameters[0].get_displayname(0) - parsearg = converters[0].parse_arg(argname, displayname) + if limited_capi: + parsearg = None + else: + displayname = parameters[0].get_displayname(0) + parsearg = converters[0].parse_arg(argname, displayname) if parsearg is None: parsearg = """ if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ @@ -1250,21 +1265,24 @@ def parser_body( parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') - elif not requires_defining_class and pos_only == len(parameters) - pseudo_args and clinic.limited_capi: + elif (not requires_defining_class and pos_only == len(parameters) and + not pseudo_args and limited_capi): # positional-only for the limited C API flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_code = [normalize_snippet(""" - if (!PyArg_ParseTuple(args, "{format_units}:{name}", - {parse_arguments})) - goto exit; - """, indent=4)] - argname_fmt = 'args[%d]' - declarations = "" - - parser_definition = parser_body(parser_prototype, *parser_code, - declarations=declarations) + if all(isinstance(c, object_converter) and c.format_unit == 'O' for c in converters): + parser_code = [normalize_snippet(""" + if (!PyArg_UnpackTuple(args, "{name}", %d, %d, + {parse_arguments})) + goto exit; + """ % (min_pos, max_pos), indent=4)] + else: + parser_code = [normalize_snippet(""" + if (!PyArg_ParseTuple(args, "{format_units}:{name}", + {parse_arguments})) + goto exit; + """, indent=4)] + parser_definition = parser_body(parser_prototype, *parser_code) elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: if not new_or_init: @@ -1284,7 +1302,6 @@ def parser_body( nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' - left_args = f"{nargs} - {max_pos}" max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos parser_code = [normalize_snippet(""" @@ -1384,7 +1401,7 @@ def parser_body( ) nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" - if clinic.limited_capi: + if limited_capi: # positional-or-keyword arguments flags = "METH_VARARGS|METH_KEYWORDS" @@ -1394,8 +1411,13 @@ def parser_body( {parse_arguments})) goto exit; """, indent=4)] - argname_fmt = 'args[%d]' - declarations = "" + declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, None, + limited_capi=limited_capi) + parser_code.append(code) elif not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" @@ -1433,92 +1455,95 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt) - parser_code.append(code) + if not limited_capi: + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, + limited_capi=limited_capi) + parser_code.append(code) - add_label: str | None = None - for i, p in enumerate(parameters): - if isinstance(p.converter, defining_class_converter): - raise ValueError("defining_class should be the first " - "parameter (after self)") - displayname = p.get_displayname(i+1) - parsearg = p.converter.parse_arg(argname_fmt % i, displayname) - if parsearg is None: - parser_code = None - break - if add_label and (i == pos_only or i == max_pos): - parser_code.append("%s:" % add_label) - add_label = None - if not p.is_optional(): - parser_code.append(normalize_snippet(parsearg, indent=4)) - elif i < pos_only: - add_label = 'skip_optional_posonly' - parser_code.append(normalize_snippet(""" - if (nargs < %d) {{ - goto %s; - }} - """ % (i + 1, add_label), indent=4)) - if has_optional_kw: - parser_code.append(normalize_snippet(""" - noptargs--; - """, indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=4)) - else: - if i < max_pos: - label = 'skip_optional_pos' - first_opt = max(min_pos, pos_only) - else: - label = 'skip_optional_kwonly' - first_opt = max_pos + min_kw_only - if vararg != NO_VARARG: - first_opt += 1 - if i == first_opt: - add_label = label + add_label: str | None = None + for i, p in enumerate(parameters): + if isinstance(p.converter, defining_class_converter): + raise ValueError("defining_class should be the first " + "parameter (after self)") + displayname = p.get_displayname(i+1) + parsearg = p.converter.parse_arg(argname_fmt % i, displayname) + if parsearg is None: + parser_code = None + break + if add_label and (i == pos_only or i == max_pos): + parser_code.append("%s:" % add_label) + add_label = None + if not p.is_optional(): + parser_code.append(normalize_snippet(parsearg, indent=4)) + elif i < pos_only: + add_label = 'skip_optional_posonly' parser_code.append(normalize_snippet(""" - if (!noptargs) {{ + if (nargs < %d) {{ goto %s; }} - """ % add_label, indent=4)) - if i + 1 == len(parameters): + """ % (i + 1, add_label), indent=4)) + if has_optional_kw: + parser_code.append(normalize_snippet(""" + noptargs--; + """, indent=4)) parser_code.append(normalize_snippet(parsearg, indent=4)) else: - add_label = label - parser_code.append(normalize_snippet(""" - if (%s) {{ - """ % (argname_fmt % i), indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=8)) - parser_code.append(normalize_snippet(""" - if (!--noptargs) {{ + if i < max_pos: + label = 'skip_optional_pos' + first_opt = max(min_pos, pos_only) + else: + label = 'skip_optional_kwonly' + first_opt = max_pos + min_kw_only + if vararg != NO_VARARG: + first_opt += 1 + if i == first_opt: + add_label = label + parser_code.append(normalize_snippet(""" + if (!noptargs) {{ goto %s; }} - }} - """ % add_label, indent=4)) + """ % add_label, indent=4)) + if i + 1 == len(parameters): + parser_code.append(normalize_snippet(parsearg, indent=4)) + else: + add_label = label + parser_code.append(normalize_snippet(""" + if (%s) {{ + """ % (argname_fmt % i), indent=4)) + parser_code.append(normalize_snippet(parsearg, indent=8)) + parser_code.append(normalize_snippet(""" + if (!--noptargs) {{ + goto %s; + }} + }} + """ % add_label, indent=4)) - if parser_code is not None: - if add_label: - parser_code.append("%s:" % add_label) - else: - declarations = declare_parser(f, hasformat=True) - if not new_or_init: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] + if parser_code is not None: + if add_label: + parser_code.append("%s:" % add_label) else: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, None) - parser_code.append(code) + declarations = declare_parser(f, hasformat=True) + if not new_or_init: + parser_code = [normalize_snippet(""" + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + parser_code = [normalize_snippet(""" + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, None, + limited_capi=limited_capi) + parser_code.append(code) if deprecated_positionals: code = self.deprecate_positional_use(f, deprecated_positionals) @@ -3098,6 +3123,8 @@ class CConverter(metaclass=CConverterAutoRegister): # "#include "name" // reason" include: tuple[str, str] | None = None + broken_limited_capi: bool = False + # keep in sync with self_converter.__init__! def __init__(self, # Positional args: @@ -3262,7 +3289,7 @@ def parse_argument(self, args: list[str]) -> None: elif self.subclass_of: args.append(self.subclass_of) - s = ("&" if self.parse_by_reference else "") + self.name + s = ("&" if self.parse_by_reference else "") + self.parser_name args.append(s) if self.length: From 4116592b6f014a2720e9b09e2c8dec4bf4b4cd8f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 28 Aug 2023 15:32:07 +0200 Subject: [PATCH 364/598] gh-108278: Deprecate passing the three first params as keyword args for sqlite3 UDF creation APIs (#108281) Deprecate passing name, number of arguments, and the callable as keyword arguments, for the following sqlite3.Connection APIs: - create_function(name, nargs, callable, ...) - create_aggregate(name, nargs, callable) The affected parameters will become positional-only in Python 3.15. --- Doc/library/sqlite3.rst | 10 ++++ Doc/whatsnew/3.13.rst | 11 +++- Lib/test/test_sqlite3/test_userfunctions.py | 23 ++++++++ ...-08-22-13-51-10.gh-issue-108278.11d_qG.rst | 9 +++ Modules/_sqlite/clinic/connection.c.h | 59 ++++++++++++++++++- Modules/_sqlite/connection.c | 6 +- 6 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-22-13-51-10.gh-issue-108278.11d_qG.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 0b53f97b284f07..0d7abae6de2694 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -763,6 +763,11 @@ Connection objects ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) + .. versionchanged:: 3.13 + + Passing *name*, *narg*, and *func* as keyword arguments is deprecated. + These parameters will become positional-only in Python 3.15. + .. method:: create_aggregate(name, n_arg, aggregate_class) @@ -817,6 +822,11 @@ Connection objects 3 + .. versionchanged:: 3.13 + + Passing *name*, *n_arg*, and *aggregate_class* as keyword arguments is deprecated. + These parameters will become positional-only in Python 3.15. + .. method:: create_window_function(name, num_params, aggregate_class, /) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d31d458b2fb9c9..a17b549aec5d8f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -252,7 +252,16 @@ Deprecated * Passing more than one positional argument to :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is deprecated. The remaining parameters will become keyword-only in Python 3.15. - (Contributed by Erlend E. Aasland in :gh:`107948`.) + + Deprecate passing name, number of arguments, and the callable as keyword + arguments, for the following :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.create_function` + * :meth:`~sqlite3.Connection.create_aggregate` + + The affected parameters will become positional-only in Python 3.15. + + (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) Pending Removal in Python 3.14 ------------------------------ diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 5d12636dcd2b63..d86b8c6025f671 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -421,6 +421,29 @@ def test_func_return_illegal_value(self): self.assertRaisesRegex(sqlite.OperationalError, msg, self.con.execute, "select badreturn()") + def test_func_keyword_args(self): + regex = ( + r"Passing keyword arguments 'name', 'narg' and 'func' to " + r"_sqlite3.Connection.create_function\(\) is deprecated. " + r"Parameters 'name', 'narg' and 'func' will become " + r"positional-only in Python 3.15." + ) + + def noop(): + return None + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function("noop", 0, func=noop) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function("noop", narg=0, func=noop) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function(name="noop", narg=0, func=noop) + self.assertEqual(cm.filename, __file__) + class WindowSumInt: def __init__(self): diff --git a/Misc/NEWS.d/next/Library/2023-08-22-13-51-10.gh-issue-108278.11d_qG.rst b/Misc/NEWS.d/next/Library/2023-08-22-13-51-10.gh-issue-108278.11d_qG.rst new file mode 100644 index 00000000000000..85bedc1f3f852d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-22-13-51-10.gh-issue-108278.11d_qG.rst @@ -0,0 +1,9 @@ +Deprecate passing name, number of arguments, and the callable as keyword +arguments, for the following :class:`sqlite3.Connection` APIs: + +* :meth:`~sqlite3.Connection.create_function` +* :meth:`~sqlite3.Connection.create_aggregate` + +The affected parameters will become positional-only in Python 3.15. + +Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 75a7df437bfb80..992b2e6c72ea56 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -418,7 +418,12 @@ PyDoc_STRVAR(pysqlite_connection_create_function__doc__, "create_function($self, /, name, narg, func, *, deterministic=False)\n" "--\n" "\n" -"Creates a new function."); +"Creates a new function.\n" +"\n" +"Note: Passing keyword arguments \'name\', \'narg\' and \'func\' to\n" +"_sqlite3.Connection.create_function() is deprecated. Parameters\n" +"\'name\', \'narg\' and \'func\' will become positional-only in Python 3.15.\n" +""); #define PYSQLITE_CONNECTION_CREATE_FUNCTION_METHODDEF \ {"create_function", _PyCFunction_CAST(pysqlite_connection_create_function), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_function__doc__}, @@ -429,6 +434,17 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, int narg, PyObject *func, int deterministic); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.create_function'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.create_function'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.create_function'." +# endif +#endif + static PyObject * pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { @@ -469,6 +485,16 @@ pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls if (!args) { goto exit; } + if (nargs < 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'name', 'narg' and 'func' to " + "_sqlite3.Connection.create_function() is deprecated. Parameters " + "'name', 'narg' and 'func' will become positional-only in Python " + "3.15.", 1)) + { + goto exit; + } + } if (!PyUnicode_Check(args[0])) { _PyArg_BadArgument("create_function", "argument 'name'", "str", args[0]); goto exit; @@ -582,7 +608,13 @@ PyDoc_STRVAR(pysqlite_connection_create_aggregate__doc__, "create_aggregate($self, /, name, n_arg, aggregate_class)\n" "--\n" "\n" -"Creates a new aggregate."); +"Creates a new aggregate.\n" +"\n" +"Note: Passing keyword arguments \'name\', \'n_arg\' and \'aggregate_class\'\n" +"to _sqlite3.Connection.create_aggregate() is deprecated. Parameters\n" +"\'name\', \'n_arg\' and \'aggregate_class\' will become positional-only in\n" +"Python 3.15.\n" +""); #define PYSQLITE_CONNECTION_CREATE_AGGREGATE_METHODDEF \ {"create_aggregate", _PyCFunction_CAST(pysqlite_connection_create_aggregate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_aggregate__doc__}, @@ -593,6 +625,17 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, const char *name, int n_arg, PyObject *aggregate_class); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.create_aggregate'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.create_aggregate'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.create_aggregate'." +# endif +#endif + static PyObject * pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { @@ -631,6 +674,16 @@ pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cl if (!args) { goto exit; } + if (nargs < 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' " + "to _sqlite3.Connection.create_aggregate() is deprecated. " + "Parameters 'name', 'n_arg' and 'aggregate_class' will become " + "positional-only in Python 3.15.", 1)) + { + goto exit; + } + } if (!PyUnicode_Check(args[0])) { _PyArg_BadArgument("create_aggregate", "argument 'name'", "str", args[0]); goto exit; @@ -1681,4 +1734,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=bc31bec42067a8bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f80eb1d02cf698e4 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index a7780dedb55794..45ec0f99237b7f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1139,6 +1139,7 @@ _sqlite3.Connection.create_function as pysqlite_connection_create_function name: str narg: int func: object + / [from 3.15] * deterministic: bool = False @@ -1150,7 +1151,7 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, PyTypeObject *cls, const char *name, int narg, PyObject *func, int deterministic) -/*[clinic end generated code: output=8a811529287ad240 input=b3e8e1d8ddaffbef]*/ +/*[clinic end generated code: output=8a811529287ad240 input=c7c313b0ca8b519e]*/ { int rc; int flags = SQLITE_UTF8; @@ -1341,6 +1342,7 @@ _sqlite3.Connection.create_aggregate as pysqlite_connection_create_aggregate name: str n_arg: int aggregate_class: object + / [from 3.15] Creates a new aggregate. [clinic start generated code]*/ @@ -1350,7 +1352,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, PyTypeObject *cls, const char *name, int n_arg, PyObject *aggregate_class) -/*[clinic end generated code: output=1b02d0f0aec7ff96 input=68a2a26366d4c686]*/ +/*[clinic end generated code: output=1b02d0f0aec7ff96 input=8087056db6eae1cf]*/ { int rc; From 47d7eba889bc03884744f978f5f8612380363332 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 28 Aug 2023 10:17:00 -0700 Subject: [PATCH 365/598] gh-108487: Move assert(self != NULL) down beyond DEOPT_IF() (#108510) --- .../2023-08-26-04-31-01.gh-issue-108487.1Gbr9k.rst | 1 + Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-26-04-31-01.gh-issue-108487.1Gbr9k.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-26-04-31-01.gh-issue-108487.1Gbr9k.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-26-04-31-01.gh-issue-108487.1Gbr9k.rst new file mode 100644 index 00000000000000..277b7c067e2528 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-26-04-31-01.gh-issue-108487.1Gbr9k.rst @@ -0,0 +1 @@ +Move an assert that would cause a spurious crash in a devious case that should only trigger deoptimization. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a55460afea751c..93926c03421eb7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3346,9 +3346,9 @@ dummy_func( inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); - assert(self != NULL); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3f46f1a10a247f..5940c185817604 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4412,9 +4412,9 @@ callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); - assert(self != NULL); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { From 242bef459bfbd0ec5e45e6d47df2709093cfefa7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 28 Aug 2023 19:25:16 +0100 Subject: [PATCH 366/598] gh-108494: Argument clinic: Improve the `parse_file()` API (#108575) Co-authored-by: Victor Stinner --- Lib/test/test_clinic.py | 26 ++++++-------------------- Tools/clinic/clinic.py | 14 +++++++------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index fcccf988d3a7dd..0592c2e3a2b291 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -13,7 +13,6 @@ import os.path import re import sys -import types import unittest test_tools.skip_if_missing('clinic') @@ -22,16 +21,9 @@ from clinic import DSLParser -def default_namespace(): - ns = types.SimpleNamespace() - ns.force = False - ns.limited_capi = clinic.DEFAULT_LIMITED_CAPI - return ns - - def _make_clinic(*, filename='clinic_tests'): clang = clinic.CLanguage(None) - c = clinic.Clinic(clang, filename=filename) + c = clinic.Clinic(clang, filename=filename, limited_capi=False) c.block_parser = clinic.BlockParser('', clang) return c @@ -60,11 +52,6 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, return cm.exception -class MockClinic: - def __init__(self): - self.limited_capi = clinic.DEFAULT_LIMITED_CAPI - - class ClinicWholeFileTest(TestCase): maxDiff = None @@ -138,7 +125,7 @@ def test_parse_with_body_prefix(self): clang.body_prefix = "//" clang.start_line = "//[{dsl_name} start]" clang.stop_line = "//[{dsl_name} stop]" - cl = clinic.Clinic(clang, filename="test.c") + cl = clinic.Clinic(clang, filename="test.c", limited_capi=False) raw = dedent(""" //[clinic start] //module test @@ -704,9 +691,8 @@ def expect_parsing_failure( self, *, filename, expected_error, verify=True, output=None ): errmsg = re.escape(dedent(expected_error).strip()) - ns = default_namespace() with self.assertRaisesRegex(clinic.ClinicError, errmsg): - clinic.parse_file(filename, ns=ns) + clinic.parse_file(filename, limited_capi=False) def test_parse_file_no_extension(self) -> None: self.expect_parsing_failure( @@ -846,9 +832,9 @@ def _test(self, input, output): blocks = list(clinic.BlockParser(input, language)) writer = clinic.BlockPrinter(language) - mock_clinic = MockClinic() + c = _make_clinic() for block in blocks: - writer.print_block(block, clinic=mock_clinic) + writer.print_block(block, clinic=c) output = writer.f.getvalue() assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) @@ -874,7 +860,7 @@ def test_round_trip_2(self): def _test_clinic(self, input, output): language = clinic.CLanguage(None) - c = clinic.Clinic(language, filename="file") + c = clinic.Clinic(language, filename="file", limited_capi=False) c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) computed = c.parse(input) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 723592fa266c4c..39a776f02b5bf5 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -63,7 +63,6 @@ version = '1' -DEFAULT_LIMITED_CAPI = False NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" CLINIC_PREFIXED_ARGS = { @@ -2414,8 +2413,8 @@ def __init__( printer: BlockPrinter | None = None, *, filename: str, + limited_capi: bool, verify: bool = True, - limited_capi: bool = False, ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) @@ -2612,11 +2611,10 @@ def __repr__(self) -> str: def parse_file( filename: str, *, - ns: argparse.Namespace, + limited_capi: bool, output: str | None = None, + verify: bool = True, ) -> None: - verify = not ns.force - limited_capi = ns.limited_capi if not output: output = filename @@ -6190,7 +6188,8 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: continue if ns.verbose: print(path) - parse_file(path, ns=ns) + parse_file(path, + verify=not ns.force, limited_capi=ns.limited_capi) return if not ns.filename: @@ -6202,7 +6201,8 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: for filename in ns.filename: if ns.verbose: print(filename) - parse_file(filename, output=ns.output, ns=ns) + parse_file(filename, output=ns.output, + verify=not ns.force, limited_capi=ns.limited_capi) def main(argv: list[str] | None = None) -> NoReturn: From f75cefd402c4c830228d85ca3442377ebaf09454 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 28 Aug 2023 20:31:03 +0200 Subject: [PATCH 367/598] gh-106670: Allow Pdb to move between chained exceptions (#106676) --- Doc/library/pdb.rst | 53 ++- Doc/whatsnew/3.13.rst | 7 + Lib/pdb.py | 142 +++++++- Lib/test/test_pdb.py | 343 ++++++++++++++++++ ...-07-12-10-59-08.gh-issue-106670.goQ2Sy.rst | 1 + 5 files changed, 526 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-12-10-59-08.gh-issue-106670.goQ2Sy.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index ef52370bff8058..3aaac15ee5780c 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -175,8 +175,8 @@ slightly different way: .. function:: pm() - Enter post-mortem debugging of the traceback found in - :data:`sys.last_traceback`. + Enter post-mortem debugging of the exception found in + :data:`sys.last_exc`. The ``run*`` functions and :func:`set_trace` are aliases for instantiating the @@ -639,6 +639,55 @@ can be overridden by the local file. Print the return value for the last return of the current function. +.. pdbcommand:: exceptions [excnumber] + + List or jump between chained exceptions. + + When using ``pdb.pm()`` or ``Pdb.post_mortem(...)`` with a chained exception + instead of a traceback, it allows the user to move between the + chained exceptions using ``exceptions`` command to list exceptions, and + ``exception `` to switch to that exception. + + + Example:: + + def out(): + try: + middle() + except Exception as e: + raise ValueError("reraise middle() error") from e + + def middle(): + try: + return inner(0) + except Exception as e: + raise ValueError("Middle fail") + + def inner(x): + 1 / x + + out() + + calling ``pdb.pm()`` will allow to move between exceptions:: + + > example.py(5)out() + -> raise ValueError("reraise middle() error") from e + + (Pdb) exceptions + 0 ZeroDivisionError('division by zero') + 1 ValueError('Middle fail') + > 2 ValueError('reraise middle() error') + + (Pdb) exceptions 0 + > example.py(16)inner() + -> 1 / x + + (Pdb) up + > example.py(10)middle() + -> return inner(0) + + .. versionadded:: 3.13 + .. rubric:: Footnotes .. [1] Whether a frame is considered to originate in a certain module diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a17b549aec5d8f..1c94da23bc1b70 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -158,6 +158,13 @@ pathlib :meth:`~pathlib.Path.is_dir`. (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) +pdb +--- + +* Add ability to move between chained exceptions during post mortem debugging in :func:`~pdb.pm` using + the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias + Bussonnier in :gh:`106676`.) + sqlite3 ------- diff --git a/Lib/pdb.py b/Lib/pdb.py index 3db3e6a5be1a7b..90f26a2eb99848 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -85,6 +85,7 @@ import traceback import linecache +from contextlib import contextmanager from typing import Union @@ -205,10 +206,15 @@ def namespace(self): # line_prefix = ': ' # Use this to get the old situation back line_prefix = '\n-> ' # Probably a better default -class Pdb(bdb.Bdb, cmd.Cmd): + +class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None + # Limit the maximum depth of chained exceptions, we should be handling cycles, + # but in case there are recursions, we stop at 999. + MAX_CHAINED_EXCEPTION_DEPTH = 999 + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) @@ -256,6 +262,9 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.commands_bnum = None # The breakpoint number for which we are # defining a list + self._chained_exceptions = tuple() + self._chained_exception_index = 0 + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -414,7 +423,64 @@ def preloop(self): self.message('display %s: %r [old: %r]' % (expr, newvalue, oldvalue)) - def interaction(self, frame, traceback): + def _get_tb_and_exceptions(self, tb_or_exc): + """ + Given a tracecack or an exception, return a tuple of chained exceptions + and current traceback to inspect. + + This will deal with selecting the right ``__cause__`` or ``__context__`` + as well as handling cycles, and return a flattened list of exceptions we + can jump to with do_exceptions. + + """ + _exceptions = [] + if isinstance(tb_or_exc, BaseException): + traceback, current = tb_or_exc.__traceback__, tb_or_exc + + while current is not None: + if current in _exceptions: + break + _exceptions.append(current) + if current.__cause__ is not None: + current = current.__cause__ + elif ( + current.__context__ is not None and not current.__suppress_context__ + ): + current = current.__context__ + + if len(_exceptions) >= self.MAX_CHAINED_EXCEPTION_DEPTH: + self.message( + f"More than {self.MAX_CHAINED_EXCEPTION_DEPTH}" + " chained exceptions found, not all exceptions" + "will be browsable with `exceptions`." + ) + break + else: + traceback = tb_or_exc + return tuple(reversed(_exceptions)), traceback + + @contextmanager + def _hold_exceptions(self, exceptions): + """ + Context manager to ensure proper cleaning of exceptions references + + When given a chained exception instead of a traceback, + pdb may hold references to many objects which may leak memory. + + We use this context manager to make sure everything is properly cleaned + + """ + try: + self._chained_exceptions = exceptions + self._chained_exception_index = len(exceptions) - 1 + yield + finally: + # we can't put those in forget as otherwise they would + # be cleared on exception change + self._chained_exceptions = tuple() + self._chained_exception_index = 0 + + def interaction(self, frame, tb_or_exc): # Restore the previous signal handler at the Pdb prompt. if Pdb._previous_sigint_handler: try: @@ -423,14 +489,17 @@ def interaction(self, frame, traceback): pass else: Pdb._previous_sigint_handler = None - if self.setup(frame, traceback): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") + + _chained_exceptions, tb = self._get_tb_and_exceptions(tb_or_exc) + with self._hold_exceptions(_chained_exceptions): + if self.setup(frame, tb): + # no interaction desired at this time (happens if .pdbrc contains + # a command like "continue") + self.forget() + return + self.print_stack_entry(self.stack[self.curindex]) + self._cmdloop() self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) - self._cmdloop() - self.forget() def displayhook(self, obj): """Custom displayhook for the exec in default(), which prevents @@ -1073,6 +1142,44 @@ def _select_frame(self, number): self.print_stack_entry(self.stack[self.curindex]) self.lineno = None + def do_exceptions(self, arg): + """exceptions [number] + + List or change current exception in an exception chain. + + Without arguments, list all the current exception in the exception + chain. Exceptions will be numbered, with the current exception indicated + with an arrow. + + If given an integer as argument, switch to the exception at that index. + """ + if not self._chained_exceptions: + self.message( + "Did not find chained exceptions. To move between" + " exceptions, pdb/post_mortem must be given an exception" + " object rather than a traceback." + ) + return + if not arg: + for ix, exc in enumerate(self._chained_exceptions): + prompt = ">" if ix == self._chained_exception_index else " " + rep = repr(exc) + if len(rep) > 80: + rep = rep[:77] + "..." + self.message(f"{prompt} {ix:>3} {rep}") + else: + try: + number = int(arg) + except ValueError: + self.error("Argument must be an integer") + return + if 0 <= number < len(self._chained_exceptions): + self._chained_exception_index = number + self.setup(None, self._chained_exceptions[number].__traceback__) + self.print_stack_entry(self.stack[self.curindex]) + else: + self.error("No exception with that number") + def do_up(self, arg): """u(p) [count] @@ -1890,11 +1997,15 @@ def set_trace(*, header=None): # Post-Mortem interface def post_mortem(t=None): - """Enter post-mortem debugging of the given *traceback* object. + """Enter post-mortem debugging of the given *traceback*, or *exception* + object. If no traceback is given, it uses the one of the exception that is currently being handled (an exception must be being handled if the default is to be used). + + If `t` is an exception object, the `exceptions` command makes it possible to + list and inspect its chained exceptions (if any). """ # handling the default if t is None: @@ -1911,12 +2022,8 @@ def post_mortem(t=None): p.interaction(None, t) def pm(): - """Enter post-mortem debugging of the traceback found in sys.last_traceback.""" - if hasattr(sys, 'last_exc'): - tb = sys.last_exc.__traceback__ - else: - tb = sys.last_traceback - post_mortem(tb) + """Enter post-mortem debugging of the traceback found in sys.last_exc.""" + post_mortem(sys.last_exc) # Main program for testing @@ -1996,8 +2103,7 @@ def main(): traceback.print_exc() print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") - t = e.__traceback__ - pdb.interaction(None, t) + pdb.interaction(None, e) print("Post mortem debugger finished. The " + target + " will be restarted") diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index a66953557e52dc..734b5c83cdff7d 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -826,6 +826,349 @@ def test_convenience_variables(): (Pdb) continue """ + +def test_post_mortem_chained(): + """Test post mortem traceback debugging of chained exception + + >>> def test_function_2(): + ... try: + ... 1/0 + ... finally: + ... print('Exception!') + + >>> def test_function_reraise(): + ... try: + ... test_function_2() + ... except ZeroDivisionError as e: + ... raise ZeroDivisionError('reraised') from e + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... try: + ... test_function_reraise() + ... except Exception as e: + ... # same as pdb.post_mortem(e), but with custom pdb instance. + ... instance.reset() + ... instance.interaction(None, e) + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'exceptions', + ... 'exceptions 0', + ... 'up', + ... 'down', + ... 'exceptions 1', + ... 'up', + ... 'down', + ... 'exceptions -1', + ... 'exceptions 3', + ... 'up', + ... 'exit', + ... ]): + ... try: + ... test_function() + ... except ZeroDivisionError: + ... print('Correctly reraised.') + Exception! + > (5)test_function_reraise() + -> raise ZeroDivisionError('reraised') from e + (Pdb) exceptions + 0 ZeroDivisionError('division by zero') + > 1 ZeroDivisionError('reraised') + (Pdb) exceptions 0 + > (3)test_function_2() + -> 1/0 + (Pdb) up + > (3)test_function_reraise() + -> test_function_2() + (Pdb) down + > (3)test_function_2() + -> 1/0 + (Pdb) exceptions 1 + > (5)test_function_reraise() + -> raise ZeroDivisionError('reraised') from e + (Pdb) up + > (5)test_function() + -> test_function_reraise() + (Pdb) down + > (5)test_function_reraise() + -> raise ZeroDivisionError('reraised') from e + (Pdb) exceptions -1 + *** No exception with that number + (Pdb) exceptions 3 + *** No exception with that number + (Pdb) up + > (5)test_function() + -> test_function_reraise() + (Pdb) exit + """ + + +def test_post_mortem_cause_no_context(): + """Test post mortem traceback debugging of chained exception + + >>> def main(): + ... try: + ... raise ValueError('Context Not Shown') + ... except Exception as e1: + ... raise ValueError("With Cause") from TypeError('The Cause') + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... try: + ... main() + ... except Exception as e: + ... # same as pdb.post_mortem(e), but with custom pdb instance. + ... instance.reset() + ... instance.interaction(None, e) + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'exceptions', + ... 'exceptions 1', + ... 'up', + ... 'down', + ... 'exit', + ... ]): + ... try: + ... test_function() + ... except ValueError: + ... print('Ok.') + > (5)main() + -> raise ValueError("With Cause") from TypeError('The Cause') + (Pdb) exceptions + 0 TypeError('The Cause') + > 1 ValueError('With Cause') + (Pdb) exceptions 1 + > (5)main() + -> raise ValueError("With Cause") from TypeError('The Cause') + (Pdb) up + > (5)test_function() + -> main() + (Pdb) down + > (5)main() + -> raise ValueError("With Cause") from TypeError('The Cause') + (Pdb) exit""" + + +def test_post_mortem_context_of_the_cause(): + """Test post mortem traceback debugging of chained exception + + + >>> def main(): + ... try: + ... raise TypeError('Context of the cause') + ... except Exception as e1: + ... try: + ... raise ValueError('Root Cause') + ... except Exception as e2: + ... ex = e2 + ... raise ValueError("With Cause, and cause has context") from ex + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... try: + ... main() + ... except Exception as e: + ... # same as pdb.post_mortem(e), but with custom pdb instance. + ... instance.reset() + ... instance.interaction(None, e) + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'exceptions', + ... 'exceptions 2', + ... 'up', + ... 'down', + ... 'exceptions 3', + ... 'up', + ... 'down', + ... 'exceptions 4', + ... 'up', + ... 'down', + ... 'exit', + ... ]): + ... try: + ... test_function() + ... except ValueError: + ... print('Correctly reraised.') + > (9)main() + -> raise ValueError("With Cause, and cause has context") from ex + (Pdb) exceptions + 0 TypeError('Context of the cause') + 1 ValueError('Root Cause') + > 2 ValueError('With Cause, and cause has context') + (Pdb) exceptions 2 + > (9)main() + -> raise ValueError("With Cause, and cause has context") from ex + (Pdb) up + > (5)test_function() + -> main() + (Pdb) down + > (9)main() + -> raise ValueError("With Cause, and cause has context") from ex + (Pdb) exceptions 3 + *** No exception with that number + (Pdb) up + > (5)test_function() + -> main() + (Pdb) down + > (9)main() + -> raise ValueError("With Cause, and cause has context") from ex + (Pdb) exceptions 4 + *** No exception with that number + (Pdb) up + > (5)test_function() + -> main() + (Pdb) down + > (9)main() + -> raise ValueError("With Cause, and cause has context") from ex + (Pdb) exit + """ + + +def test_post_mortem_from_none(): + """Test post mortem traceback debugging of chained exception + + In particular that cause from None (which sets __supress_context__ to True) + does not show context. + + + >>> def main(): + ... try: + ... raise TypeError('Context of the cause') + ... except Exception as e1: + ... raise ValueError("With Cause, and cause has context") from None + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... try: + ... main() + ... except Exception as e: + ... # same as pdb.post_mortem(e), but with custom pdb instance. + ... instance.reset() + ... instance.interaction(None, e) + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'exceptions', + ... 'exit', + ... ]): + ... try: + ... test_function() + ... except ValueError: + ... print('Correctly reraised.') + > (5)main() + -> raise ValueError("With Cause, and cause has context") from None + (Pdb) exceptions + > 0 ValueError('With Cause, and cause has context') + (Pdb) exit + """ + + +def test_post_mortem_complex(): + """Test post mortem traceback debugging of chained exception + + Test with simple and complex cycles, exception groups,... + + >>> def make_ex_with_stack(type_, *content, from_=None): + ... try: + ... raise type_(*content) from from_ + ... except Exception as out: + ... return out + ... + + >>> def cycle(): + ... try: + ... raise ValueError("Cycle Leaf") + ... except Exception as e: + ... raise e from e + ... + + >>> def tri_cycle(): + ... a = make_ex_with_stack(ValueError, "Cycle1") + ... b = make_ex_with_stack(ValueError, "Cycle2") + ... c = make_ex_with_stack(ValueError, "Cycle3") + ... + ... a.__cause__ = b + ... b.__cause__ = c + ... + ... raise c from a + ... + + >>> def cause(): + ... try: + ... raise ValueError("Cause Leaf") + ... except Exception as e: + ... raise e + ... + + >>> def context(n=10): + ... try: + ... raise ValueError(f"Context Leaf {n}") + ... except Exception as e: + ... if n == 0: + ... raise ValueError(f"With Context {n}") from e + ... else: + ... context(n - 1) + ... + + >>> def main(): + ... try: + ... cycle() + ... except Exception as e1: + ... try: + ... tri_cycle() + ... except Exception as e2: + ... ex = e2 + ... raise ValueError("With Context and With Cause") from ex + + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... try: + ... main() + ... except Exception as e: + ... # same as pdb.post_mortem(e), but with custom pdb instance. + ... instance.reset() + ... instance.interaction(None, e) + + >>> with PdbTestInput( # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... ["exceptions", + ... "exceptions 0", + ... "exceptions 1", + ... "exceptions 2", + ... "exceptions 3", + ... "exit"], + ... ): + ... try: + ... test_function() + ... except ValueError: + ... print('Correctly reraised.') + > (9)main() + -> raise ValueError("With Context and With Cause") from ex + (Pdb) exceptions + 0 ValueError('Cycle2') + 1 ValueError('Cycle1') + 2 ValueError('Cycle3') + > 3 ValueError('With Context and With Cause') + (Pdb) exceptions 0 + > (3)make_ex_with_stack() + -> raise type_(*content) from from_ + (Pdb) exceptions 1 + > (3)make_ex_with_stack() + -> raise type_(*content) from from_ + (Pdb) exceptions 2 + > (3)make_ex_with_stack() + -> raise type_(*content) from from_ + (Pdb) exceptions 3 + > (9)main() + -> raise ValueError("With Context and With Cause") from ex + (Pdb) exit + """ + + def test_post_mortem(): """Test post mortem traceback debugging. diff --git a/Misc/NEWS.d/next/Library/2023-07-12-10-59-08.gh-issue-106670.goQ2Sy.rst b/Misc/NEWS.d/next/Library/2023-07-12-10-59-08.gh-issue-106670.goQ2Sy.rst new file mode 100644 index 00000000000000..0bb18312a673fc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-12-10-59-08.gh-issue-106670.goQ2Sy.rst @@ -0,0 +1 @@ +Add the new ``exceptions`` command to the Pdb debugger. It makes it possible to move between chained exceptions when using post mortem debugging. From cf7ba83eb274df8389cb9ebdf8601132c47de48a Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 28 Aug 2023 23:04:12 +0300 Subject: [PATCH 368/598] gh-108455: Run `mypy` on `Tools/peg_generator` (#108456) Co-authored-by: Hugo van Kemenade --- .github/workflows/mypy.yml | 7 ++++++- Tools/peg_generator/mypy.ini | 6 +++++- Tools/peg_generator/pegen/build.py | 7 ++++--- Tools/peg_generator/pegen/keywordgen.py | 1 + Tools/peg_generator/pegen/parser.py | 11 ++++++----- Tools/requirements-dev.txt | 2 +- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 75264c8e746663..82714c20e968a9 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -9,6 +9,7 @@ on: paths: - "Tools/clinic/**" - "Tools/cases_generator/**" + - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - ".github/workflows/mypy.yml" workflow_dispatch: @@ -29,7 +30,11 @@ jobs: mypy: strategy: matrix: - target: ["Tools/cases_generator", "Tools/clinic"] + target: [ + "Tools/cases_generator", + "Tools/clinic", + "Tools/peg_generator", + ] name: Run mypy on ${{ matrix.target }} runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 17323722c85ad0..186e633ea4eb2d 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -1,5 +1,6 @@ [mypy] -files = pegen +files = Tools/peg_generator/pegen +pretty = True follow_imports = error no_implicit_optional = True @@ -24,3 +25,6 @@ show_error_codes = True [mypy-pegen.grammar_parser] strict_optional = False + +[mypy-setuptools.*] +ignore_missing_imports = True diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index e41330fade94e3..81efcadc001ec5 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -5,7 +5,7 @@ import sysconfig import tempfile import tokenize -from typing import IO, Dict, List, Optional, Set, Tuple +from typing import IO, Any, Dict, List, Optional, Set, Tuple from pegen.c_generator import CParserGenerator from pegen.grammar import Grammar @@ -18,6 +18,7 @@ MOD_DIR = pathlib.Path(__file__).resolve().parent TokenDefinitions = Tuple[Dict[int, str], Dict[str, int], Set[str]] +Incomplete = Any # TODO: install `types-setuptools` and remove this alias def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[str]: @@ -28,7 +29,7 @@ def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[ return f"{flags} {py_flags_nodist}".split() -def fixup_build_ext(cmd): +def fixup_build_ext(cmd: Incomplete) -> None: """Function needed to make build_ext tests pass. When Python was built with --enable-shared on Unix, -L. is not enough to @@ -74,7 +75,7 @@ def compile_c_extension( keep_asserts: bool = True, disable_optimization: bool = False, library_dir: Optional[str] = None, -) -> str: +) -> pathlib.Path: """Compile the generated source for a parser generator into an extension module. The extension module will be generated in the same directory as the provided path diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py index bbf13267e5763b..82d717b72976e5 100644 --- a/Tools/peg_generator/pegen/keywordgen.py +++ b/Tools/peg_generator/pegen/keywordgen.py @@ -35,6 +35,7 @@ issoftkeyword = frozenset(softkwlist).__contains__ '''.lstrip() + def main() -> None: parser = argparse.ArgumentParser( description="Generate the Lib/keywords.py file from the grammar." diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index 034e8e6017a22d..eab48efe2089ee 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -10,7 +10,6 @@ from pegen.tokenizer import Mark, Tokenizer, exact_token_types T = TypeVar("T") -P = TypeVar("P", bound="Parser") F = TypeVar("F", bound=Callable[..., Any]) @@ -21,7 +20,7 @@ def logger(method: F) -> F: """ method_name = method.__name__ - def logger_wrapper(self: P, *args: object) -> T: + def logger_wrapper(self: "Parser", *args: object) -> Any: if not self._verbose: return method(self, *args) argsr = ",".join(repr(arg) for arg in args) @@ -41,7 +40,7 @@ def memoize(method: F) -> F: """Memoize a symbol method.""" method_name = method.__name__ - def memoize_wrapper(self: P, *args: object) -> T: + def memoize_wrapper(self: "Parser", *args: object) -> Any: mark = self._mark() key = mark, method_name, args # Fast path: cache hit, and not verbose. @@ -74,11 +73,13 @@ def memoize_wrapper(self: P, *args: object) -> T: return cast(F, memoize_wrapper) -def memoize_left_rec(method: Callable[[P], Optional[T]]) -> Callable[[P], Optional[T]]: +def memoize_left_rec( + method: Callable[["Parser"], Optional[T]] +) -> Callable[["Parser"], Optional[T]]: """Memoize a left-recursive symbol method.""" method_name = method.__name__ - def memoize_left_rec_wrapper(self: P) -> Optional[T]: + def memoize_left_rec_wrapper(self: "Parser") -> Optional[T]: mark = self._mark() key = mark, method_name, () # Fast path: cache hit, and not verbose. diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 111773f4f47378..b814169974bd08 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,3 +1,3 @@ # Requirements file for external linters and checks we run on -# Tools/clinic and Tools/cases_generator/ in CI +# Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI mypy==1.5.1 From 0bd2ba553d1cbf4137295dad21fc06e7745ab858 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 28 Aug 2023 22:31:45 +0100 Subject: [PATCH 369/598] GH-107603: Clinic: Pass specific attributes to ``print_block()`` (#108581) --- Lib/test/test_clinic.py | 2 +- Tools/clinic/clinic.py | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 0592c2e3a2b291..b29f2ea55516fe 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -834,7 +834,7 @@ def _test(self, input, output): writer = clinic.BlockPrinter(language) c = _make_clinic() for block in blocks: - writer.print_block(block, clinic=c) + writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes) output = writer.f.getvalue() assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 39a776f02b5bf5..2b016685e84ce5 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2172,8 +2172,9 @@ def print_block( self, block: Block, *, - clinic: Clinic, core_includes: bool = False, + limited_capi: bool, + header_includes: dict[str, str], ) -> None: input = block.input output = block.output @@ -2203,7 +2204,7 @@ def print_block( output = '' if core_includes: - if not clinic.limited_capi: + if not limited_capi: output += textwrap.dedent(""" #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head @@ -2212,12 +2213,10 @@ def print_block( """) - if clinic is not None: - # Emit optional includes - for include, reason in sorted(clinic.includes.items()): - line = f'#include "{include}"' - line = line.ljust(35) + f'// {reason}\n' - output += line + # Emit optional "#include" directives for C headers + for include, reason in sorted(header_includes.items()): + line = f'#include "{include}"'.ljust(35) + f'// {reason}\n' + output += line input = ''.join(block.input) output += ''.join(block.output) @@ -2531,7 +2530,9 @@ def parse(self, input: str) -> str: self.parsers[dsl_name] = parsers[dsl_name](self) parser = self.parsers[dsl_name] parser.parse(block) - printer.print_block(block, clinic=self) + printer.print_block(block, + limited_capi=self.limited_capi, + header_includes=self.includes) # these are destinations not buffers for name, destination in self.destinations.items(): @@ -2546,7 +2547,9 @@ def parse(self, input: str) -> str: block.input = "dump " + name + "\n" warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") printer.write("\n") - printer.print_block(block, clinic=self) + printer.print_block(block, + limited_capi=self.limited_capi, + header_includes=self.includes) continue if destination.type == 'file': @@ -2571,7 +2574,10 @@ def parse(self, input: str) -> str: block.input = 'preserve\n' printer_2 = BlockPrinter(self.language) - printer_2.print_block(block, core_includes=True, clinic=self) + printer_2.print_block(block, + core_includes=True, + limited_capi=self.limited_capi, + header_includes=self.includes) write_file(destination.filename, printer_2.f.getvalue()) continue From 0b6a4cb0df28a2698cd2402f988b40309c04b666 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 01:42:24 +0200 Subject: [PATCH 370/598] gh-107149: Rename _PyUnstable_GetUnaryIntrinsicName() function (#108441) * Rename _PyUnstable_GetUnaryIntrinsicName() to PyUnstable_GetUnaryIntrinsicName() * Rename _PyUnstable_GetBinaryIntrinsicName() to PyUnstable_GetBinaryIntrinsicName(). --- Include/cpython/compile.h | 4 ++-- Modules/_opcode.c | 4 ++-- Python/intrinsics.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index ae17cef554fa17..265f5397b45116 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -78,5 +78,5 @@ PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode); -PyAPI_FUNC(PyObject*) _PyUnstable_GetUnaryIntrinsicName(int index); -PyAPI_FUNC(PyObject*) _PyUnstable_GetBinaryIntrinsicName(int index); +PyAPI_FUNC(PyObject*) PyUnstable_GetUnaryIntrinsicName(int index); +PyAPI_FUNC(PyObject*) PyUnstable_GetBinaryIntrinsicName(int index); diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 3e13dbb6edc26e..dac9c019b249e9 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -310,7 +310,7 @@ _opcode_get_intrinsic1_descs_impl(PyObject *module) return NULL; } for (int i=0; i <= MAX_INTRINSIC_1; i++) { - PyObject *name = _PyUnstable_GetUnaryIntrinsicName(i); + PyObject *name = PyUnstable_GetUnaryIntrinsicName(i); if (name == NULL) { Py_DECREF(list); return NULL; @@ -337,7 +337,7 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return NULL; } for (int i=0; i <= MAX_INTRINSIC_2; i++) { - PyObject *name = _PyUnstable_GetBinaryIntrinsicName(i); + PyObject *name = PyUnstable_GetBinaryIntrinsicName(i); if (name == NULL) { Py_DECREF(list); return NULL; diff --git a/Python/intrinsics.c b/Python/intrinsics.c index fefee0fc4ebbec..8e59c63aea39ff 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -269,7 +269,7 @@ _PyIntrinsics_BinaryFunctions[] = { #undef INTRINSIC_FUNC_ENTRY PyObject* -_PyUnstable_GetUnaryIntrinsicName(int index) +PyUnstable_GetUnaryIntrinsicName(int index) { if (index < 0 || index > MAX_INTRINSIC_1) { return NULL; @@ -278,7 +278,7 @@ _PyUnstable_GetUnaryIntrinsicName(int index) } PyObject* -_PyUnstable_GetBinaryIntrinsicName(int index) +PyUnstable_GetBinaryIntrinsicName(int index) { if (index < 0 || index > MAX_INTRINSIC_2) { return NULL; From 5c68cba268c07bac165834ad5c72448b195327c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 01:51:39 +0200 Subject: [PATCH 371/598] gh-108494: Build _testclinic_limited on Windows (#108589) Add _testclinic_limited project to the Visual Studio solution: * In "PCbuild/", copy "_asyncio.vcxproj" to "_testclinic_limited.vcxproj", replace "RootNamespace" with "_testclinic_limited", replace "_asyncio.c" with "_testclinic_limited.c". * Open Visual Studio, open "PCbuild\pcbuild.sln", add the existing "PCbuild\_testclinic_limited.vcxproj" project to the solution. * Add a dependency from "python" project to the "_testclinic_limited" project. * Save and exit Visual Studio. * Add ";_testclinic_limited" to "" in "PCbuild\pcbuild.proj". --- PCbuild/_testclinic_limited.vcxproj | 109 +++++++++++++++++ PCbuild/_testclinic_limited.vcxproj.filters | 21 ++++ PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 124 ++++++++++---------- PCbuild/readme.txt | 1 + 5 files changed, 195 insertions(+), 62 deletions(-) create mode 100644 PCbuild/_testclinic_limited.vcxproj create mode 100644 PCbuild/_testclinic_limited.vcxproj.filters diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj new file mode 100644 index 00000000000000..b00b2be491b423 --- /dev/null +++ b/PCbuild/_testclinic_limited.vcxproj @@ -0,0 +1,109 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410} + _testclinic_limited + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + \ No newline at end of file diff --git a/PCbuild/_testclinic_limited.vcxproj.filters b/PCbuild/_testclinic_limited.vcxproj.filters new file mode 100644 index 00000000000000..8ecdd99a4ac220 --- /dev/null +++ b/PCbuild/_testclinic_limited.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + {2422278e-eeeb-4241-8182-433e2bc5a7fc} + + + {41f1cd52-b682-46aa-a7fd-7bdf81a18010} + + + + + Source Files + + + + + Resource Files + + + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 7735c22f8ae1b3..b7b78be768d7ec 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -77,7 +77,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index ca70213755310c..a0b5fbd1304b41 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -10,43 +10,43 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", "{B11D750F-CD1F-4A96-85CE-E69A5C5259F9}" ProjectSection(ProjectDependencies) = postProject - {9E48B300-37D1-11DD-8C41-005056C00008} = {9E48B300-37D1-11DD-8C41-005056C00008} - {9EC7190A-249F-4180-A900-548FDCF3055F} = {9EC7190A-249F-4180-A900-548FDCF3055F} - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} = {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D} = {6901D91C-6E48-4BB7-9FEC-700C8131DF1D} - {54B1431F-B86B-4ACB-B28C-88BCF93191D8} = {54B1431F-B86B-4ACB-B28C-88BCF93191D8} - {F749B822-B489-4CA5-A3AD-CE078F5F338A} = {F749B822-B489-4CA5-A3AD-CE078F5F338A} - {D06B6426-4762-44CC-8BAD-D79052507F2F} = {D06B6426-4762-44CC-8BAD-D79052507F2F} - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} = {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} - {CB435430-EBB1-478B-8F4E-C256F6838F55} = {CB435430-EBB1-478B-8F4E-C256F6838F55} - {17E1E049-C309-4D79-843F-AE483C264AEA} = {17E1E049-C309-4D79-843F-AE483C264AEA} - {384C224A-7474-476E-A01B-750EA7DE918C} = {384C224A-7474-476E-A01B-750EA7DE918C} + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410} = {01FDF29A-40A1-46DF-84F5-85EBBD2A2410} + {0E9791DB-593A-465F-98BC-681011311617} = {0E9791DB-593A-465F-98BC-681011311617} + {0E9791DB-593A-465F-98BC-681011311618} = {0E9791DB-593A-465F-98BC-681011311618} {12728250-16EC-4DC6-94D7-E21DD88947F8} = {12728250-16EC-4DC6-94D7-E21DD88947F8} - {86937F53-C189-40EF-8CE8-8759D8E7D480} = {86937F53-C189-40EF-8CE8-8759D8E7D480} - {28B5D777-DDF2-4B6B-B34F-31D938813856} = {28B5D777-DDF2-4B6B-B34F-31D938813856} - {31FFC478-7B4A-43E8-9954-8D03E2187E9C} = {31FFC478-7B4A-43E8-9954-8D03E2187E9C} - {F9D71780-F393-11E0-BE50-0800200C9A66} = {F9D71780-F393-11E0-BE50-0800200C9A66} - {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} = {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} - {C6E20F84-3247-4AD6-B051-B073268F73BA} = {C6E20F84-3247-4AD6-B051-B073268F73BA} - {B244E787-C445-441C-BDF4-5A4F1A3A1E51} = {B244E787-C445-441C-BDF4-5A4F1A3A1E51} - {18CAE28C-B454-46C1-87A0-493D91D97F03} = {18CAE28C-B454-46C1-87A0-493D91D97F03} {13CECB97-4119-4316-9D42-8534019A5A44} = {13CECB97-4119-4316-9D42-8534019A5A44} - {885D4898-D08D-4091-9C40-C700CFE3FC5A} = {885D4898-D08D-4091-9C40-C700CFE3FC5A} + {16BFE6F0-22EF-40B5-B831-7E937119EF10} = {16BFE6F0-22EF-40B5-B831-7E937119EF10} + {17E1E049-C309-4D79-843F-AE483C264AEA} = {17E1E049-C309-4D79-843F-AE483C264AEA} + {18CAE28C-B454-46C1-87A0-493D91D97F03} = {18CAE28C-B454-46C1-87A0-493D91D97F03} + {2097F1C1-597C-4167-93E3-656A7D6339B2} = {2097F1C1-597C-4167-93E3-656A7D6339B2} + {28B5D777-DDF2-4B6B-B34F-31D938813856} = {28B5D777-DDF2-4B6B-B34F-31D938813856} + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} = {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} + {384C224A-7474-476E-A01B-750EA7DE918C} = {384C224A-7474-476E-A01B-750EA7DE918C} {447F05A8-F581-4CAC-A466-5AC7936E207E} = {447F05A8-F581-4CAC-A466-5AC7936E207E} - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} = {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} = {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} - {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} = {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} + {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} = {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} + {54B1431F-B86B-4ACB-B28C-88BCF93191D8} = {54B1431F-B86B-4ACB-B28C-88BCF93191D8} + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D} = {6901D91C-6E48-4BB7-9FEC-700C8131DF1D} + {6DAC66D9-E703-4624-BE03-49112AB5AA62} = {6DAC66D9-E703-4624-BE03-49112AB5AA62} {73FCD2BD-F133-46B7-8EC1-144CD82A59D5} = {73FCD2BD-F133-46B7-8EC1-144CD82A59D5} - {2097F1C1-597C-4167-93E3-656A7D6339B2} = {2097F1C1-597C-4167-93E3-656A7D6339B2} - {A2697BD3-28C1-4AEC-9106-8B748639FD16} = {A2697BD3-28C1-4AEC-9106-8B748639FD16} + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} = {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} + {86937F53-C189-40EF-8CE8-8759D8E7D480} = {86937F53-C189-40EF-8CE8-8759D8E7D480} + {885D4898-D08D-4091-9C40-C700CFE3FC5A} = {885D4898-D08D-4091-9C40-C700CFE3FC5A} {900342D7-516A-4469-B1AD-59A66E49A25F} = {900342D7-516A-4469-B1AD-59A66E49A25F} - {6DAC66D9-E703-4624-BE03-49112AB5AA62} = {6DAC66D9-E703-4624-BE03-49112AB5AA62} - {0E9791DB-593A-465F-98BC-681011311617} = {0E9791DB-593A-465F-98BC-681011311617} - {0E9791DB-593A-465F-98BC-681011311618} = {0E9791DB-593A-465F-98BC-681011311618} + {9E48B300-37D1-11DD-8C41-005056C00008} = {9E48B300-37D1-11DD-8C41-005056C00008} + {9EC7190A-249F-4180-A900-548FDCF3055F} = {9EC7190A-249F-4180-A900-548FDCF3055F} + {A2697BD3-28C1-4AEC-9106-8B748639FD16} = {A2697BD3-28C1-4AEC-9106-8B748639FD16} + {A840DDFB-ED50-484B-B527-B32E7CF90FD5} = {A840DDFB-ED50-484B-B527-B32E7CF90FD5} + {B244E787-C445-441C-BDF4-5A4F1A3A1E51} = {B244E787-C445-441C-BDF4-5A4F1A3A1E51} + {C6E20F84-3247-4AD6-B051-B073268F73BA} = {C6E20F84-3247-4AD6-B051-B073268F73BA} + {CB435430-EBB1-478B-8F4E-C256F6838F55} = {CB435430-EBB1-478B-8F4E-C256F6838F55} + {D06B6426-4762-44CC-8BAD-D79052507F2F} = {D06B6426-4762-44CC-8BAD-D79052507F2F} {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E} = {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E} - {16BFE6F0-22EF-40B5-B831-7E937119EF10} = {16BFE6F0-22EF-40B5-B831-7E937119EF10} + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} = {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} + {F749B822-B489-4CA5-A3AD-CE078F5F338A} = {F749B822-B489-4CA5-A3AD-CE078F5F338A} + {F9D71780-F393-11E0-BE50-0800200C9A66} = {F9D71780-F393-11E0-BE50-0800200C9A66} {FCBE1EF2-E0F0-40B1-88B5-00A35D378742} = {FCBE1EF2-E0F0-40B1-88B5-00A35D378742} - {A840DDFB-ED50-484B-B527-B32E7CF90FD5} = {A840DDFB-ED50-484B-B527-B32E7CF90FD5} + {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} = {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" @@ -157,6 +157,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_uuid", "_uuid.vcxproj", "{ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_wmi", "_wmi.vcxproj", "{54B1431F-B86B-4ACB-B28C-88BCF93191D8}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testclinic_limited", "_testclinic_limited.vcxproj", "{01FDF29A-40A1-46DF-84F5-85EBBD2A2410}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -433,38 +435,6 @@ Global {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.ActiveCfg = Release|x64 {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.Build.0 = Release|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|ARM.ActiveCfg = Debug|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|ARM.Build.0 = Debug|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|ARM64.Build.0 = Debug|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.ActiveCfg = Debug|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.Build.0 = Debug|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.ActiveCfg = Debug|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.Build.0 = Debug|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|ARM.Build.0 = PGInstrument|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|ARM.Build.0 = PGUpdate|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|ARM.ActiveCfg = Release|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|ARM.Build.0 = Release|ARM - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|ARM64.ActiveCfg = Release|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|ARM64.Build.0 = Release|ARM64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.ActiveCfg = Release|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.Build.0 = Release|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.ActiveCfg = Release|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.Build.0 = Release|x64 {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|ARM.ActiveCfg = Debug|ARM {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|ARM.Build.0 = Debug|ARM {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|ARM64.ActiveCfg = Debug|ARM64 @@ -1646,6 +1616,38 @@ Global {54B1431F-B86B-4ACB-B28C-88BCF93191D8}.Release|Win32.Build.0 = Release|Win32 {54B1431F-B86B-4ACB-B28C-88BCF93191D8}.Release|x64.ActiveCfg = Release|x64 {54B1431F-B86B-4ACB-B28C-88BCF93191D8}.Release|x64.Build.0 = Release|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|ARM.ActiveCfg = Debug|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|ARM.Build.0 = Debug|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|ARM64.Build.0 = Debug|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|Win32.ActiveCfg = Debug|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|Win32.Build.0 = Debug|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|x64.ActiveCfg = Debug|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Debug|x64.Build.0 = Debug|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|ARM.ActiveCfg = Release|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|ARM.Build.0 = Release|ARM + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|ARM64.ActiveCfg = Release|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|ARM64.Build.0 = Release|ARM64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|Win32.ActiveCfg = Release|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|Win32.Build.0 = Release|Win32 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.ActiveCfg = Release|x64 + {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 1d3b45b7912a34..199aacdf7687ed 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -144,6 +144,7 @@ _socket _testbuffer _testcapi _testclinic +_testclinic_limited _testconsole _testimportmultiple _testmultiphase From 88f1c5b454c34efc167a94b5e2d67ec042834e5b Mon Sep 17 00:00:00 2001 From: xzmeng Date: Tue, 29 Aug 2023 08:14:21 +0800 Subject: [PATCH 372/598] Fix misc doc typos (#108592) --- Doc/c-api/stable.rst | 2 +- Doc/c-api/typeobj.rst | 2 +- Doc/library/idle.rst | 2 +- Doc/library/importlib.rst | 2 +- Doc/library/sqlite3.rst | 2 +- Doc/library/statistics.rst | 2 +- Doc/library/tkinter.rst | 2 +- Doc/using/configure.rst | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index c66b296d304adc..63a100a6f26f24 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -18,7 +18,7 @@ way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, but will need to be compiled separately for 3.9.x and 3.10.x. -There are two tiers of C API with different stability exepectations: +There are two tiers of C API with different stability expectations: - :ref:`Unstable API `, may change in minor versions without a deprecation period. It is marked by the ``PyUnstable`` prefix in names. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index acaf0eced35b3f..f417c68fd1efd4 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1728,7 +1728,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) treated as read-only. Some types may not store their dictionary in this slot. - Use :c:func:`PyType_GetDict` to retreive the dictionary for an arbitrary + Use :c:func:`PyType_GetDict` to retrieve the dictionary for an arbitrary type. .. versionchanged:: 3.12 diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 3058bcead661f3..3211da50dc745c 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -479,7 +479,7 @@ Search and Replace Any selection becomes a search target. However, only selections within a line work because searches are only performed within lines with the -terminal newline removed. If ``[x] Regular expresion`` is checked, the +terminal newline removed. If ``[x] Regular expression`` is checked, the target is interpreted according to the Python re module. .. _completions: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 1d378dbbdace5d..d8cc7079a1d812 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1270,7 +1270,7 @@ an :term:`importer`. You can get the same effect as this function by implementing the basic interface of multi-phase init (:pep:`489`) and lying about - support for mulitple interpreters (or per-interpreter GIL). + support for multiple interpreters (or per-interpreter GIL). .. warning:: Using this function to disable the check can lead to diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 0d7abae6de2694..5137e173e23740 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1566,7 +1566,7 @@ Cursor objects :raises ProgrammingError: If *sql* contains more than one SQL statement, - or is not a DML statment. + or is not a DML statement. Example: diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 368b2a17cef997..a8a79012565321 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1091,7 +1091,7 @@ parameter, ``h``, representing the variance of the kernel function. import math def kde_normal(sample, h): - "Create a continous probability density function from a sample." + "Create a continuous probability density function from a sample." # Smooth the sample with a normal distribution of variance h. kernel_h = NormalDist(0.0, math.sqrt(h)).pdf n = len(sample) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 9f6c3e3e862c42..76cccc4e9dc9bc 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -348,7 +348,7 @@ Understanding How Tkinter Wraps Tcl/Tk When your application uses Tkinter's classes and methods, internally Tkinter is assembling strings representing Tcl/Tk commands, and executing those -commands in the Tcl interpreter attached to your applicaton's :class:`Tk` +commands in the Tcl interpreter attached to your application's :class:`Tk` instance. Whether it's trying to navigate reference documentation, trying to find diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 50f60ee54fed14..6279b0efbf9321 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -230,7 +230,7 @@ Install Options Install architecture-independent files in PREFIX. On Unix, it defaults to :file:`/usr/local`. - This value can be retrived at runtime using :data:`sys.prefix`. + This value can be retrieved at runtime using :data:`sys.prefix`. As an example, one can use ``--prefix="$HOME/.local/"`` to install a Python in its home directory. @@ -239,7 +239,7 @@ Install Options Install architecture-dependent files in EPREFIX, defaults to :option:`--prefix`. - This value can be retrived at runtime using :data:`sys.exec_prefix`. + This value can be retrieved at runtime using :data:`sys.exec_prefix`. .. cmdoption:: --disable-test-modules From 4fb96a11db5eaca3646bfa697d191469e567d283 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 02:29:46 +0200 Subject: [PATCH 373/598] gh-106320: Remove private _Py_Identifier API (#108593) Remove the private _Py_Identifier type and related private functions from the public C API: * _PyObject_GetAttrId() * _PyObject_LookupSpecialId() * _PyObject_SetAttrId() * _PyType_LookupId() * _Py_IDENTIFIER() * _Py_static_string() * _Py_static_string_init() Move them to the internal C API: add a new pycore_identifier.h header file. No longer export these functions. --- Include/cpython/object.h | 42 ------------------- Include/internal/pycore_call.h | 1 + Include/internal/pycore_dict.h | 1 + Include/internal/pycore_identifier.h | 54 +++++++++++++++++++++++++ Include/internal/pycore_unicodeobject.h | 1 + Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 5 ++- PCbuild/pythoncore.vcxproj.filters | 7 +++- 8 files changed, 66 insertions(+), 46 deletions(-) create mode 100644 Include/internal/pycore_identifier.h diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 5f8b1f7c195501..489b2eecd32991 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -19,43 +19,6 @@ PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GetRefTotal(PyInterpreterState *); #endif -/********************* String Literals ****************************************/ -/* This structure helps managing static strings. The basic usage goes like this: - Instead of doing - - r = PyObject_CallMethod(o, "foo", "args", ...); - - do - - _Py_IDENTIFIER(foo); - ... - r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); - - PyId_foo is a static variable, either on block level or file level. On first - usage, the string "foo" is interned, and the structures are linked. On interpreter - shutdown, all strings are released. - - Alternatively, _Py_static_string allows choosing the variable name. - _PyUnicode_FromId returns a borrowed reference to the interned string. - _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. -*/ -typedef struct _Py_Identifier { - const char* string; - // Index in PyInterpreterState.unicode.ids.array. It is process-wide - // unique and must be initialized to -1. - Py_ssize_t index; -} _Py_Identifier; - -#ifndef Py_BUILD_CORE -// For now we are keeping _Py_IDENTIFIER for continued use -// in non-builtin extensions (and naughty PyPI modules). - -#define _Py_static_string_init(value) { .string = (value), .index = -1 } -#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) -#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) - -#endif /* !Py_BUILD_CORE */ - typedef struct { /* Number implementations must check *both* arguments for proper type and implement the necessary conversions @@ -273,8 +236,6 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); @@ -282,9 +243,6 @@ PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); -PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); - PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index db719234e05bf3..8846155b38defb 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_identifier.h" // _Py_Identifier #include "pycore_pystate.h" // _PyThreadState_GET() /* Suggested size (number of positional arguments) for arrays of PyObject* diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index e79fb207cf75b1..47b5948f66343a 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,6 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_identifier.h" // _Py_Identifier #include "pycore_object.h" // PyDictOrValues // Unsafe flavor of PyDict_GetItemWithError(): no error checking diff --git a/Include/internal/pycore_identifier.h b/Include/internal/pycore_identifier.h new file mode 100644 index 00000000000000..0e015a40c831f4 --- /dev/null +++ b/Include/internal/pycore_identifier.h @@ -0,0 +1,54 @@ +/* String Literals: _Py_Identifier API */ + +#ifndef Py_INTERNAL_IDENTIFIER_H +#define Py_INTERNAL_IDENTIFIER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* This structure helps managing static strings. The basic usage goes like this: + Instead of doing + + r = PyObject_CallMethod(o, "foo", "args", ...); + + do + + _Py_IDENTIFIER(foo); + ... + r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); + + PyId_foo is a static variable, either on block level or file level. On first + usage, the string "foo" is interned, and the structures are linked. On interpreter + shutdown, all strings are released. + + Alternatively, _Py_static_string allows choosing the variable name. + _PyUnicode_FromId returns a borrowed reference to the interned string. + _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. +*/ +typedef struct _Py_Identifier { + const char* string; + // Index in PyInterpreterState.unicode.ids.array. It is process-wide + // unique and must be initialized to -1. + Py_ssize_t index; +} _Py_Identifier; + +// For now we are keeping _Py_IDENTIFIER for continued use +// in non-builtin extensions (and naughty PyPI modules). + +#define _Py_static_string_init(value) { .string = (value), .index = -1 } +#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) +#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) + +extern PyObject* _PyType_LookupId(PyTypeObject *, _Py_Identifier *); +extern PyObject* _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); +extern PyObject* _PyObject_GetAttrId(PyObject *, _Py_Identifier *); +extern int _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_IDENTIFIER_H diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 08c3dfec93bf05..360a9e1819f8e8 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_fileutils.h" // _Py_error_handler +#include "pycore_identifier.h" // _Py_Identifier #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI /* --- Characters Type APIs ----------------------------------------------- */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 54b64e123cd7bc..62a55bb2478fe4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1769,6 +1769,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ $(srcdir)/Include/internal/pycore_hamt.h \ $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_identifier.h \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_interp.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b0e62864421e17..bc1250f63957e2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -235,6 +235,7 @@ + @@ -248,7 +249,7 @@ - + @@ -280,7 +281,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d5f61e9c5d7c89..a8a11d3b8b83b1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -609,6 +609,9 @@ Include\internal + + Include\internal + Include\internal @@ -650,7 +653,7 @@ Include\internal - + Include\internal @@ -737,7 +740,7 @@ Include\internal - + Modules\zlib From 39506ee565ce819e14e8071227fc53b8dcb6a788 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 03:20:02 +0200 Subject: [PATCH 374/598] gh-108240: Add pycore_capsule.h internal header file (#108596) Move _PyCapsule_SetTraverse() definition to a new internal header file: pycore_capsule.h. --- Include/internal/pycore_capsule.h | 17 +++++++++++++++++ Include/pycapsule.h | 5 ----- Makefile.pre.in | 1 + Modules/socketmodule.c | 1 + Objects/capsule.c | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 7 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 Include/internal/pycore_capsule.h diff --git a/Include/internal/pycore_capsule.h b/Include/internal/pycore_capsule.h new file mode 100644 index 00000000000000..aa2c67f3a8f002 --- /dev/null +++ b/Include/internal/pycore_capsule.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_PYCAPSULE_H +#define Py_INTERNAL_PYCAPSULE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Export for '_socket' shared extension +PyAPI_FUNC(int) _PyCapsule_SetTraverse(PyObject *op, traverseproc traverse_func, inquiry clear_func); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYCAPSULE_H */ diff --git a/Include/pycapsule.h b/Include/pycapsule.h index 8660e83088edd6..666b9f86739670 100644 --- a/Include/pycapsule.h +++ b/Include/pycapsule.h @@ -48,15 +48,10 @@ PyAPI_FUNC(int) PyCapsule_SetName(PyObject *capsule, const char *name); PyAPI_FUNC(int) PyCapsule_SetContext(PyObject *capsule, void *context); -#ifdef Py_BUILD_CORE -PyAPI_FUNC(int) _PyCapsule_SetTraverse(PyObject *op, traverseproc traverse_func, inquiry clear_func); -#endif - PyAPI_FUNC(void *) PyCapsule_Import( const char *name, /* UTF-8 encoded string */ int no_block); - #ifdef __cplusplus } #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 62a55bb2478fe4..310caa94b9e476 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1743,6 +1743,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_bytes_methods.h \ $(srcdir)/Include/internal/pycore_bytesobject.h \ $(srcdir)/Include/internal/pycore_call.h \ + $(srcdir)/Include/internal/pycore_capsule.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_ceval_state.h \ $(srcdir)/Include/internal/pycore_code.h \ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8cc45e22549f30..e3681853dad095 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -106,6 +106,7 @@ Local naming conventions: #endif #include "Python.h" +#include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_dict.h" // _PyDict_Pop() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState diff --git a/Objects/capsule.c b/Objects/capsule.c index a1abcf683f0133..555979dab2b789 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -1,6 +1,7 @@ /* Wrap void * pointers to be passed between C modules */ #include "Python.h" +#include "pycore_capsule.h" // export _PyCapsule_SetTraverse() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK() diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index bc1250f63957e2..9ab84702d1f2d2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -206,6 +206,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a8a11d3b8b83b1..d5c53a0c64b29c 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -528,6 +528,9 @@ Include\internal + + Include\internal + Include\internal From 301eb7e607be6ea15c7a1e792f1cb927ba423932 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 04:05:11 +0200 Subject: [PATCH 375/598] gh-106320: Remove _PyAnextAwaitable_Type from the public C API (#108597) It's not needed to declare it in Include/iterobject.h: just use "extern" where it's used (only in object.c). --- Include/iterobject.h | 3 --- Objects/object.c | 7 ++++--- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 1 - Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Include/iterobject.h b/Include/iterobject.h index fff30f7176fdeb..e69d09719bb4d1 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,9 +7,6 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; -#ifdef Py_BUILD_CORE -extern PyTypeObject _PyAnextAwaitable_Type; -#endif #define PySeqIter_Check(op) Py_IS_TYPE((op), &PySeqIter_Type) diff --git a/Objects/object.c b/Objects/object.c index 02e2611020983d..0d88421bf0fdc9 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2049,11 +2049,12 @@ _PyObject_InitState(PyInterpreterState *interp) } -extern PyTypeObject _Py_GenericAliasIterType; -extern PyTypeObject _PyMemoryIter_Type; +extern PyTypeObject _PyAnextAwaitable_Type; +extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject _PyLineIterator; +extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyPositionsIterator; -extern PyTypeObject _PyLegacyEventHandler_Type; +extern PyTypeObject _Py_GenericAliasIterType; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 55cc2d37e1ebc7..18d1b1a5d0a37f 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -69,7 +69,6 @@ def preprocess(filename, cwd = os.path.abspath(cwd or '.') filename = _normpath(filename, cwd) - print(filename) postargs = POST_ARGS if os.path.basename(filename) not in USE_LIMITED_C_API: postargs += ('-DPy_BUILD_CORE=1',) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 58f9c8e8a3e9a5..bb85fba895bc25 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -57,6 +57,7 @@ Objects/genobject.c - _PyCoroWrapper_Type - Objects/interpreteridobject.c - PyInterpreterID_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - +Objects/iterobject.c - _PyAnextAwaitable_Type - Objects/listobject.c - PyListIter_Type - Objects/listobject.c - PyListRevIter_Type - Objects/listobject.c - PyList_Type - @@ -70,6 +71,7 @@ Objects/moduleobject.c - PyModule_Type - Objects/namespaceobject.c - _PyNamespace_Type - Objects/object.c - _PyNone_Type - Objects/object.c - _PyNotImplemented_Type - +Objects/object.c - _PyAnextAwaitable_Type - Objects/odictobject.c - PyODictItems_Type - Objects/odictobject.c - PyODictIter_Type - Objects/odictobject.c - PyODictKeys_Type - @@ -113,8 +115,6 @@ Objects/codeobject.c - _PyLineIterator - Objects/codeobject.c - _PyPositionsIterator - Objects/genericaliasobject.c - _Py_GenericAliasIterType - # Not in a .h file: -Objects/iterobject.c - _PyAnextAwaitable_Type - -# Not in a .h file: Objects/memoryobject.c - _PyMemoryIter_Type - Objects/unicodeobject.c - _PyUnicodeASCIIIter_Type - Objects/unionobject.c - _PyUnion_Type - From 15c5a50797020944e9ccb72bf3e28c3cca2e35c3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 04:18:52 +0200 Subject: [PATCH 376/598] gh-106320: Remove private pythonrun API (#108599) Remove these private functions from the public C API: * _PyRun_AnyFileObject() * _PyRun_InteractiveLoopObject() * _PyRun_SimpleFileObject() * _Py_SourceAsString() Move them to the internal C API: add a new pycore_pythonrun.h header file. No longer export these functions. --- Include/cpython/pythonrun.h | 24 ------------------ Include/internal/pycore_pythonrun.h | 39 +++++++++++++++++++++++++++++ Makefile.pre.in | 1 + Modules/main.c | 1 + Modules/symtablemodule.c | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ Python/bltinmodule.c | 1 + Python/pythonrun.c | 1 + 9 files changed, 48 insertions(+), 24 deletions(-) create mode 100644 Include/internal/pycore_pythonrun.h diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index 3b2537e01b83b1..edc40952254029 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -3,21 +3,11 @@ #endif PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); -PyAPI_FUNC(int) _PyRun_SimpleFileObject( - FILE *fp, - PyObject *filename, - int closeit, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_AnyFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ int closeit, PyCompilerFlags *flags); -PyAPI_FUNC(int) _PyRun_AnyFileObject( - FILE *fp, - PyObject *filename, - int closeit, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_SimpleFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ @@ -35,10 +25,6 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags); -PyAPI_FUNC(int) _PyRun_InteractiveLoopObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, @@ -69,15 +55,6 @@ PyAPI_FUNC(PyObject *) Py_CompileStringObject( #define Py_CompileString(str, p, s) Py_CompileStringExFlags((str), (p), (s), NULL, -1) #define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags((str), (p), (s), (f), -1) - -PyAPI_FUNC(const char *) _Py_SourceAsString( - PyObject *cmd, - const char *funcname, - const char *what, - PyCompilerFlags *cf, - PyObject **cmd_copy); - - /* A function flavor is also exported by libpython. It is required when libpython is accessed directly rather than using header files which defines macros below. On Windows, for example, PyAPI_FUNC() uses dllexport to @@ -114,7 +91,6 @@ PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject #define PyRun_FileFlags(fp, p, s, g, l, flags) \ PyRun_FileExFlags((fp), (p), (s), (g), (l), 0, (flags)) - /* Stuff with no proper home (yet) */ PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h new file mode 100644 index 00000000000000..0bfc5704dc4c59 --- /dev/null +++ b/Include/internal/pycore_pythonrun.h @@ -0,0 +1,39 @@ +#ifndef Py_INTERNAL_PYTHONRUN_H +#define Py_INTERNAL_PYTHONRUN_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern int _PyRun_SimpleFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); + +extern int _PyRun_AnyFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); + +extern int _PyRun_InteractiveLoopObject( + FILE *fp, + PyObject *filename, + PyCompilerFlags *flags); + +extern const char* _Py_SourceAsString( + PyObject *cmd, + const char *funcname, + const char *what, + PyCompilerFlags *cf, + PyObject **cmd_copy); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_PYTHONRUN_H + diff --git a/Makefile.pre.in b/Makefile.pre.in index 310caa94b9e476..ad3e26c036408e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1795,6 +1795,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pymem.h \ $(srcdir)/Include/internal/pycore_pymem_init.h \ $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pythonrun.h \ $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_range.h \ $(srcdir)/Include/internal/pycore_runtime.h \ diff --git a/Modules/main.c b/Modules/main.c index a5c009321d607f..7f88c97207475b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -7,6 +7,7 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pythonrun.h" // _PyRun_AnyFileObject() /* Includes for exit_sigint() */ #include // perror() diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 1cc319cc3410d8..dba80034d310af 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_pythonrun.h" // _Py_SourceAsString() #include "pycore_symtable.h" // struct symtable #include "clinic/symtablemodule.c.h" diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9ab84702d1f2d2..0b0f5ad0fdb9cd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -259,6 +259,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d5c53a0c64b29c..c70b8017935d73 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -681,6 +681,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ac9bc72fe80d6c..30dd717b0fea29 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -12,6 +12,7 @@ #include "pycore_object.h" // _Py_AddToAllObjects() #include "pycore_pyerrors.h" // _PyErr_NoMemory() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pythonrun.h" // _Py_SourceAsString() #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_tuple.h" // _PyTuple_FromArray() diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a2b50c021ce9e4..231ac78146a0e6 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -21,6 +21,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pythonrun.h" // define _PyRun_InteractiveLoopObject() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() From 8d8bf0b5140a7e5bd1cba7c66694cf44cdf6d0a9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 04:36:50 +0200 Subject: [PATCH 377/598] gh-106320: Remove private _Py_UniversalNewlineFgetsWithSize() (#108602) The remove private _Py_UniversalNewlineFgetsWithSize() function from the public C API: move it the internal C API (pycore_fileutils.h). No longer export the function. --- Include/cpython/fileobject.h | 1 - Include/internal/pycore_fileutils.h | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/cpython/fileobject.h b/Include/cpython/fileobject.h index 6fbe69604af40c..e2d89c522bdd13 100644 --- a/Include/cpython/fileobject.h +++ b/Include/cpython/fileobject.h @@ -3,7 +3,6 @@ #endif PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *); -PyAPI_FUNC(char *) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); /* The std printer acts as a preliminary sys.stderr until the new io infrastructure is in place. */ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index f6e625870de9d1..4707a9b1ff3052 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -315,6 +315,8 @@ extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootE // Export for 'select' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); +extern char* _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); + #ifdef __cplusplus } #endif From 07cf33ef24053eb101f56f0c311164b59d50157b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 04:38:23 +0200 Subject: [PATCH 378/598] gh-106320: Remove private _PyThread_at_fork_reinit() function (#108601) Move the private function to the internal C API (pycore_pythread.h) and no longer exports it. --- Include/cpython/pythread.h | 7 ------- Include/internal/pycore_pythread.h | 8 ++++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Include/cpython/pythread.h b/Include/cpython/pythread.h index ce4ec8f65b15ea..cd2aab72d52df3 100644 --- a/Include/cpython/pythread.h +++ b/Include/cpython/pythread.h @@ -4,13 +4,6 @@ #define PYTHREAD_INVALID_THREAD_ID ((unsigned long)-1) -#ifdef HAVE_FORK -/* Private function to reinitialize a lock at fork in the child process. - Reset the lock to the unlocked state. - Return 0 on success, return -1 on error. */ -PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); -#endif /* HAVE_FORK */ - #ifdef HAVE_PTHREAD_H /* Darwin needs pthread.h to know type name the pthread_key_t. */ # include diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 44846c0fc4b4c3..5ec2abda91e86b 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -75,6 +75,14 @@ struct _pythread_runtime_state { }; +#ifdef HAVE_FORK +/* Private function to reinitialize a lock at fork in the child process. + Reset the lock to the unlocked state. + Return 0 on success, return -1 on error. */ +extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); +#endif /* HAVE_FORK */ + + #ifdef __cplusplus } #endif From 21a7420190778fb6e9237bf12e029a26cd18d82d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 04:44:38 +0200 Subject: [PATCH 379/598] gh-106320: Remove private _PyGILState_GetInterpreterStateUnsafe() (#108603) The remove private _PyGILState_GetInterpreterStateUnsafe() function from the public C API: move it the internal C API (pycore_pystate.h). No longer export the function. --- Include/cpython/pystate.h | 9 --------- Include/internal/pycore_pystate.h | 8 ++++++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 2a53838314c9d7..fc5f58db86dbe8 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -227,15 +227,6 @@ PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); -/* Get the single PyInterpreterState used by this process' GILState - implementation. - - This function doesn't check for error. Return NULL before _PyGILState_Init() - is called and after _PyGILState_Fini() is called. - - See also PyInterpreterState_Get() and _PyInterpreterState_GET(). */ -PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); - /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 2e601cc6c4a464..a30036aeb57e05 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -174,6 +174,14 @@ extern int _PyOS_InterruptOccurred(PyThreadState *tstate); // Export for test_peg_generator. PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); +// Get the single PyInterpreterState used by this process' GILState +// implementation. +// +// This function doesn't check for error. Return NULL before _PyGILState_Init() +// is called and after _PyGILState_Fini() is called. +// +// See also PyInterpreterState_Get() and _PyInterpreterState_GET(). +extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); #ifdef __cplusplus } From 921eb8ebf6ae9bd359bc03c24bf1f7537bb498ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 04:59:49 +0200 Subject: [PATCH 380/598] gh-106320: Remove private _PyLong_New() function (#108604) Move the following private API to the internal C API (pycore_long.h): * _PyLong_Copy() * _PyLong_FromDigits() * _PyLong_New() No longer export most of these functions. --- Include/cpython/longintrepr.h | 8 -------- Include/internal/pycore_long.h | 11 +++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 692c69ba76db2f..fb82f83dc50e42 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -89,14 +89,6 @@ struct _longobject { _PyLongValue long_value; }; -PyAPI_FUNC(PyLongObject *) _PyLong_New(Py_ssize_t); - -/* Return a copy of src. */ -PyAPI_FUNC(PyObject *) _PyLong_Copy(PyLongObject *src); - -PyAPI_FUNC(PyLongObject *) -_PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits); - /* Inline some internals for speed. These should be in pycore_long.h * if user code didn't need them inlined. */ diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index c9d82711862895..c411ac654387c3 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -47,6 +47,17 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif +extern PyLongObject* _PyLong_New(Py_ssize_t); + +// Return a copy of src. +extern PyObject* _PyLong_Copy(PyLongObject *src); + +// Export for '_decimal' shared extension +PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( + int negative, + Py_ssize_t digit_count, + digit *digits); + /* runtime lifecycle */ From c9eefc77a7456c5354beaff9bdba449d3e42be35 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 05:13:41 +0200 Subject: [PATCH 381/598] gh-106320: Remove private _PyErr_SetKeyError() (#108607) Move the private _PyErr_SetKeyError() function to the internal C API (pycore_pyerrors.h). --- Include/cpython/pyerrors.h | 4 ---- Include/internal/pycore_pyerrors.h | 4 ++++ Objects/setobject.c | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 5c128211bd525a..cbe3be158b38be 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,10 +88,6 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif -/* Error handling definitions */ - -PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); - /* Context manipulation (PEP 3134) */ Py_DEPRECATED(3.12) PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 5d2ad50e0d2cfd..0bc20589721f35 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -16,6 +16,10 @@ extern PyObject* _PyErr_GetHandledException(PyThreadState *); extern void _PyErr_SetHandledException(PyThreadState *, PyObject *); extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); + + // Like PyErr_Format(), but saves current exception as __context__ and // __cause__. // Export for '_sqlite3' shared extension. diff --git a/Objects/setobject.c b/Objects/setobject.c index 14b53c12dda57f..6051e57731c7a3 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -36,6 +36,7 @@ #include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition #include // offsetof() From fadc2dc7df99501a40171f39b7cd23be732860cc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 05:20:31 +0200 Subject: [PATCH 382/598] gh-106320: Remove private _PyOS_IsMainThread() function (#108605) Move the following private API to the internal C API (pycore_signal.h): _PyOS_IsMainThread() and _PyOS_SigintEvent(). --- Include/internal/pycore_signal.h | 9 +++++++++ Include/intrcheck.h | 11 ++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_signal.h b/Include/internal/pycore_signal.h index 8f876759c0d07f..b38b1d30112b60 100644 --- a/Include/internal/pycore_signal.h +++ b/Include/internal/pycore_signal.h @@ -95,6 +95,15 @@ struct _signals_runtime_state { } +// Export for '_multiprocessing' shared extension +PyAPI_FUNC(int) _PyOS_IsMainThread(void); + +#ifdef MS_WINDOWS +// is not included by Python.h so use void* instead of HANDLE. +// Export for '_multiprocessing' shared extension +PyAPI_FUNC(void*) _PyOS_SigintEvent(void); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/intrcheck.h b/Include/intrcheck.h index b8cc65601683cb..1d1feee83de483 100644 --- a/Include/intrcheck.h +++ b/Include/intrcheck.h @@ -5,6 +5,7 @@ extern "C" { #endif PyAPI_FUNC(int) PyOS_InterruptOccurred(void); + #ifdef HAVE_FORK #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 PyAPI_FUNC(void) PyOS_BeforeFork(void); @@ -12,18 +13,10 @@ PyAPI_FUNC(void) PyOS_AfterFork_Parent(void); PyAPI_FUNC(void) PyOS_AfterFork_Child(void); #endif #endif + /* Deprecated, please use PyOS_AfterFork_Child() instead */ Py_DEPRECATED(3.7) PyAPI_FUNC(void) PyOS_AfterFork(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyOS_IsMainThread(void); - -#ifdef MS_WINDOWS -/* windows.h is not included by Python.h so use void* instead of HANDLE */ -PyAPI_FUNC(void*) _PyOS_SigintEvent(void); -#endif -#endif /* !Py_LIMITED_API */ - #ifdef __cplusplus } #endif From b6de2850f286455239dcf16b9682db85ae64c6a1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 05:38:51 +0200 Subject: [PATCH 383/598] gh-106320: Remove private _PyObject_GetState() (#108606) Move the private _PyObject_GetState() function to the internal C API (pycore_object.h). --- Include/internal/pycore_object.h | 4 ++++ Include/object.h | 6 ------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c6007718bae5a4..32ff2794e36231 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -442,6 +442,10 @@ extern int _PyObject_IsAbstract(PyObject *); extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); extern PyObject* _PyObject_NextNotImplemented(PyObject *); +// Pickle support. +// Export for '_datetime' shared extension +PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); + /* C function call trampolines to mitigate bad function pointer casts. * * Typical native ABIs ignore additional arguments or fill in missing diff --git a/Include/object.h b/Include/object.h index be9a0cedb7205d..d82eb6138743b9 100644 --- a/Include/object.h +++ b/Include/object.h @@ -437,12 +437,6 @@ PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *); */ PyAPI_FUNC(PyObject *) PyObject_Dir(PyObject *); -/* Pickle support. */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyObject_GetState(PyObject *); -#endif - - /* Helpers for printing recursive container types */ PyAPI_FUNC(int) Py_ReprEnter(PyObject *); PyAPI_FUNC(void) Py_ReprLeave(PyObject *); From c780698e9b8416633402657ee2846c0179705e9b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 05:40:13 +0200 Subject: [PATCH 384/598] gh-106320: Fix test_peg_generator: _Py_UniversalNewlineFgetsWithSize() (#108609) Fix test_peg_generator by exporting the _Py_UniversalNewlineFgetsWithSize() function. --- Include/internal/pycore_fileutils.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 4707a9b1ff3052..9236e5907a48d5 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -315,7 +315,8 @@ extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootE // Export for 'select' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); -extern char* _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); +// Export for test_peg_generator +PyAPI_FUNC(char*) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); #ifdef __cplusplus } From 5f85b443f7119e1c68a15fc9a342655e544d2852 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 29 Aug 2023 00:17:25 -0700 Subject: [PATCH 385/598] GH-106176, GH-104702: Fix reference leak when importing across multiple threads (#108497) --- Lib/importlib/_bootstrap.py | 115 ++++++++++++++++-- ...-08-25-14-51-06.gh-issue-106176.D1EA2a.rst | 4 + 2 files changed, 107 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-25-14-51-06.gh-issue-106176.D1EA2a.rst diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index c48fd506a0e4eb..ec2e56f6ea9ca1 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -51,18 +51,106 @@ def _new_module(name): # Module-level locking ######################################################## -# A dict mapping module names to weakrefs of _ModuleLock instances -# Dictionary protected by the global import lock +# For a list that can have a weakref to it. +class _List(list): + pass + + +# Copied from weakref.py with some simplifications and modifications unique to +# bootstrapping importlib. Many methods were simply deleting for simplicity, so if they +# are needed in the future they may work if simply copied back in. +class _WeakValueDictionary: + + def __init__(self): + self_weakref = _weakref.ref(self) + + # Inlined to avoid issues with inheriting from _weakref.ref before _weakref is + # set by _setup(). Since there's only one instance of this class, this is + # not expensive. + class KeyedRef(_weakref.ref): + + __slots__ = "key", + + def __new__(type, ob, key): + self = super().__new__(type, ob, type.remove) + self.key = key + return self + + def __init__(self, ob, key): + super().__init__(ob, self.remove) + + @staticmethod + def remove(wr): + nonlocal self_weakref + + self = self_weakref() + if self is not None: + if self._iterating: + self._pending_removals.append(wr.key) + else: + _weakref._remove_dead_weakref(self.data, wr.key) + + self._KeyedRef = KeyedRef + self.clear() + + def clear(self): + self._pending_removals = [] + self._iterating = set() + self.data = {} + + def _commit_removals(self): + pop = self._pending_removals.pop + d = self.data + while True: + try: + key = pop() + except IndexError: + return + _weakref._remove_dead_weakref(d, key) + + def get(self, key, default=None): + if self._pending_removals: + self._commit_removals() + try: + wr = self.data[key] + except KeyError: + return default + else: + if (o := wr()) is None: + return default + else: + return o + + def setdefault(self, key, default=None): + try: + o = self.data[key]() + except KeyError: + o = None + if o is None: + if self._pending_removals: + self._commit_removals() + self.data[key] = self._KeyedRef(default, key) + return default + else: + return o + + +# A dict mapping module names to weakrefs of _ModuleLock instances. +# Dictionary protected by the global import lock. _module_locks = {} -# A dict mapping thread IDs to lists of _ModuleLock instances. This maps a -# thread to the module locks it is blocking on acquiring. The values are -# lists because a single thread could perform a re-entrant import and be "in -# the process" of blocking on locks for more than one module. A thread can -# be "in the process" because a thread cannot actually block on acquiring -# more than one lock but it can have set up bookkeeping that reflects that -# it intends to block on acquiring more than one lock. -_blocking_on = {} +# A dict mapping thread IDs to weakref'ed lists of _ModuleLock instances. +# This maps a thread to the module locks it is blocking on acquiring. The +# values are lists because a single thread could perform a re-entrant import +# and be "in the process" of blocking on locks for more than one module. A +# thread can be "in the process" because a thread cannot actually block on +# acquiring more than one lock but it can have set up bookkeeping that reflects +# that it intends to block on acquiring more than one lock. +# +# The dictionary uses a WeakValueDictionary to avoid keeping unnecessary +# lists around, regardless of GC runs. This way there's no memory leak if +# the list is no longer needed (GH-106176). +_blocking_on = None class _BlockingOnManager: @@ -79,7 +167,7 @@ def __enter__(self): # re-entrant (i.e., a single thread may take it more than once) so it # wouldn't help us be correct in the face of re-entrancy either. - self.blocked_on = _blocking_on.setdefault(self.thread_id, []) + self.blocked_on = _blocking_on.setdefault(self.thread_id, _List()) self.blocked_on.append(self.lock) def __exit__(self, *args, **kwargs): @@ -1409,7 +1497,7 @@ def _setup(sys_module, _imp_module): modules, those two modules must be explicitly passed in. """ - global _imp, sys + global _imp, sys, _blocking_on _imp = _imp_module sys = sys_module @@ -1437,6 +1525,9 @@ def _setup(sys_module, _imp_module): builtin_module = sys.modules[builtin_name] setattr(self_module, builtin_name, builtin_module) + # Instantiation requires _weakref to have been set. + _blocking_on = _WeakValueDictionary() + def _install(sys_module, _imp_module): """Install importers for builtin and frozen modules""" diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-25-14-51-06.gh-issue-106176.D1EA2a.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-25-14-51-06.gh-issue-106176.D1EA2a.rst new file mode 100644 index 00000000000000..7f63d1086ea39e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-25-14-51-06.gh-issue-106176.D1EA2a.rst @@ -0,0 +1,4 @@ +Use a ``WeakValueDictionary`` to track the lists containing the modules each +thread is currently importing. This helps avoid a reference leak from +keeping the list around longer than necessary. Weakrefs are used as GC can't +interrupt the cleanup. From 6eaddc10e972273c1aed8b88c538e65e4773496e Mon Sep 17 00:00:00 2001 From: Edward Schauman-Haigh <142528725+EddInSverige@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:51:36 +0200 Subject: [PATCH 386/598] gh-108558: Improve sqlite3 row factory tests (#108578) Add test_sqlite_row_keys() to explicitly test sqlite3.Row.keys(). Cleanups: - Reduce test noise by converting docstrings to regular comments - Reduce boilerplate code by adding a setUp() method to RowFactoryTests Co-authored-by: Erlend E. Aasland --- Lib/test/test_sqlite3/test_factory.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index a7c4417862aff7..48d35b54a2e239 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -116,13 +116,16 @@ def test_is_produced_by_factory(self): class RowFactoryTests(MemoryDatabaseMixin, unittest.TestCase): + def setUp(self): + super().setUp() + self.con.row_factory = sqlite.Row + def test_custom_factory(self): self.con.row_factory = lambda cur, row: list(row) row = self.con.execute("select 1, 2").fetchone() self.assertIsInstance(row, list) def test_sqlite_row_index(self): - self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a_1, 2 as b").fetchone() self.assertIsInstance(row, sqlite.Row) @@ -153,7 +156,6 @@ def test_sqlite_row_index(self): row[complex()] # index must be int or string def test_sqlite_row_index_unicode(self): - self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) with self.assertRaises(IndexError): @@ -163,7 +165,6 @@ def test_sqlite_row_index_unicode(self): def test_sqlite_row_slice(self): # A sqlite.Row can be sliced like a list. - self.con.row_factory = sqlite.Row row = self.con.execute("select 1, 2, 3, 4").fetchone() self.assertEqual(row[0:0], ()) self.assertEqual(row[0:1], (1,)) @@ -180,8 +181,7 @@ def test_sqlite_row_slice(self): self.assertEqual(row[3:0:-2], (4, 2)) def test_sqlite_row_iter(self): - """Checks if the row object is iterable""" - self.con.row_factory = sqlite.Row + # Checks if the row object is iterable. row = self.con.execute("select 1 as a, 2 as b").fetchone() # Is iterable in correct order and produces valid results: @@ -193,23 +193,20 @@ def test_sqlite_row_iter(self): self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): - """Checks if the row object can be converted to a tuple""" - self.con.row_factory = sqlite.Row + # Checks if the row object can be converted to a tuple. row = self.con.execute("select 1 as a, 2 as b").fetchone() t = tuple(row) self.assertEqual(t, (row['a'], row['b'])) def test_sqlite_row_as_dict(self): - """Checks if the row object can be correctly converted to a dictionary""" - self.con.row_factory = sqlite.Row + # Checks if the row object can be correctly converted to a dictionary. row = self.con.execute("select 1 as a, 2 as b").fetchone() d = dict(row) self.assertEqual(d["a"], row["a"]) self.assertEqual(d["b"], row["b"]) def test_sqlite_row_hash_cmp(self): - """Checks if the row object compares and hashes correctly""" - self.con.row_factory = sqlite.Row + # Checks if the row object compares and hashes correctly. row_1 = self.con.execute("select 1 as a, 2 as b").fetchone() row_2 = self.con.execute("select 1 as a, 2 as b").fetchone() row_3 = self.con.execute("select 1 as a, 3 as b").fetchone() @@ -242,21 +239,24 @@ def test_sqlite_row_hash_cmp(self): self.assertEqual(hash(row_1), hash(row_2)) def test_sqlite_row_as_sequence(self): - """ Checks if the row object can act like a sequence """ - self.con.row_factory = sqlite.Row + # Checks if the row object can act like a sequence. row = self.con.execute("select 1 as a, 2 as b").fetchone() as_tuple = tuple(row) self.assertEqual(list(reversed(row)), list(reversed(as_tuple))) self.assertIsInstance(row, Sequence) + def test_sqlite_row_keys(self): + # Checks if the row object can return a list of columns as strings. + row = self.con.execute("select 1 as a, 2 as b").fetchone() + self.assertEqual(row.keys(), ['a', 'b']) + def test_fake_cursor_class(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # segmentation fault. # Issue #27861: Also applies for cursor factory. class FakeCursor(str): __class__ = sqlite.Cursor - self.con.row_factory = sqlite.Row self.assertRaises(TypeError, self.con.cursor, FakeCursor) self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ()) From c8847841cc5629cbceead0c09dc6f537d7b92612 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 29 Aug 2023 11:39:42 +0200 Subject: [PATCH 387/598] gh-108550: Fix sqlite3 CLI regression from gh-108551 (#108618) --- Lib/sqlite3/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 10a2e9e0a202a4..3b59763375c147 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -124,4 +124,4 @@ def main(*args): if __name__ == "__main__": - main(sys.argv) + main(sys.argv[1:]) From ecb2bf02a4a564b638f756ce6e644ec17b6edf16 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Aug 2023 13:20:32 +0300 Subject: [PATCH 388/598] gh-108617: Extend interactive session tests for sqlite3 (GH-108556) --- Lib/test/test_sqlite3/test_cli.py | 38 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index e681f5c976b7b4..303f9e03b5383f 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -87,42 +87,68 @@ def run_cli(self, *args, commands=()): def test_interact(self): out, err = self.run_cli() self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertTrue(out.endswith(self.PS1)) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 0) def test_interact_quit(self): out, err = self.run_cli(commands=(".quit",)) - self.assertIn(self.PS1, out) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertTrue(out.endswith(self.PS1)) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 0) def test_interact_version(self): out, err = self.run_cli(commands=(".version",)) self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(sqlite3.sqlite_version + "\n", out) + self.assertTrue(out.endswith(self.PS1)) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) self.assertIn(sqlite3.sqlite_version, out) def test_interact_valid_sql(self): out, err = self.run_cli(commands=("SELECT 1;",)) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("(1,)", out) + self.assertIn("(1,)\n", out) + self.assertTrue(out.endswith(self.PS1)) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_incomplete_multiline_sql(self): + out, err = self.run_cli(commands=("SELECT 1",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertTrue(out.endswith(self.PS2)) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 1) def test_interact_valid_multiline_sql(self): out, err = self.run_cli(commands=("SELECT 1\n;",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(self.PS2, out) - self.assertIn("(1,)", out) + self.assertIn("(1,)\n", out) + self.assertTrue(out.endswith(self.PS1)) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 1) def test_interact_invalid_sql(self): out, err = self.run_cli(commands=("sel;",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn("OperationalError (SQLITE_ERROR)", err) + self.assertTrue(out.endswith(self.PS1)) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) def test_interact_on_disk_file(self): self.addCleanup(unlink, TESTFN) out, err = self.run_cli(TESTFN, commands=("CREATE TABLE t(t);",)) self.assertIn(TESTFN, err) - self.assertIn(self.PS1, out) + self.assertTrue(out.endswith(self.PS1)) out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) - self.assertIn("(0,)", out) + self.assertIn("(0,)\n", out) if __name__ == "__main__": From c879b9fddf570a46d51df81fd181015ed983224a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 29 Aug 2023 11:24:06 +0100 Subject: [PATCH 389/598] gh-108455: Simplify the peg_generator mypy config (#108620) make it easier to see exactly which options from mypy's `--strict` mode can't currently be enabled --- Tools/peg_generator/mypy.ini | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 186e633ea4eb2d..bc94c6a94512b0 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -1,27 +1,17 @@ [mypy] files = Tools/peg_generator/pegen pretty = True +show_traceback = True -follow_imports = error -no_implicit_optional = True -strict_optional = True - -#check_untyped_defs = True -disallow_untyped_calls = True -disallow_untyped_defs = True - -disallow_any_generics = true -disallow_any_unimported = True -disallow_incomplete_defs = True -disallow_subclassing_any = True +# Make sure the peg_generator can be run using Python 3.10: +python_version = 3.10 -warn_unused_configs = True -warn_unused_ignores = true -warn_redundant_casts = true -warn_no_return = True +# Be strict... +strict = True -show_traceback = True -show_error_codes = True +# except for a few settings that can't yet be enabled: +warn_return_any = False +no_implicit_reexport = False [mypy-pegen.grammar_parser] strict_optional = False From 0d140b8c5ea83d095cb35b6b0c9434cb63fe6ced Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 29 Aug 2023 12:23:22 +0100 Subject: [PATCH 390/598] gh-108455: peg_generator: make the mypy config slightly stricter (#108627) * Enable `--no-implicit-reexport` * Enable the `truthy-bool` error code * Enable the `ignore-without-code` error code * Explicitly note that `--warn-unreachable` cannot yet be enabled --- Tools/peg_generator/mypy.ini | 3 ++- Tools/peg_generator/pegen/__main__.py | 5 ++++- Tools/peg_generator/pegen/parser.py | 6 +++--- Tools/peg_generator/pegen/testutil.py | 6 +++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index bc94c6a94512b0..6a45165e8eecb5 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -8,10 +8,11 @@ python_version = 3.10 # Be strict... strict = True +enable_error_code = truthy-bool,ignore-without-code # except for a few settings that can't yet be enabled: warn_return_any = False -no_implicit_reexport = False +warn_unreachable = False [mypy-pegen.grammar_parser] strict_optional = False diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 2910d6ccf1c694..262c8a6db68f6e 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -12,7 +12,10 @@ import traceback from typing import Tuple -from pegen.build import Grammar, Parser, ParserGenerator, Tokenizer +from pegen.grammar import Grammar +from pegen.parser import Parser +from pegen.parser_generator import ParserGenerator +from pegen.tokenizer import Tokenizer from pegen.validator import validate_grammar diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index eab48efe2089ee..ed0aec9db2443f 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -32,7 +32,7 @@ def logger_wrapper(self: "Parser", *args: object) -> Any: print(f"{fill}... {method_name}({argsr}) --> {tree!s:.200}") return tree - logger_wrapper.__wrapped__ = method # type: ignore + logger_wrapper.__wrapped__ = method # type: ignore[attr-defined] return cast(F, logger_wrapper) @@ -69,7 +69,7 @@ def memoize_wrapper(self: "Parser", *args: object) -> Any: self._reset(endmark) return tree - memoize_wrapper.__wrapped__ = method # type: ignore + memoize_wrapper.__wrapped__ = method # type: ignore[attr-defined] return cast(F, memoize_wrapper) @@ -153,7 +153,7 @@ def memoize_left_rec_wrapper(self: "Parser") -> Optional[T]: self._reset(endmark) return tree - memoize_left_rec_wrapper.__wrapped__ = method # type: ignore + memoize_left_rec_wrapper.__wrapped__ = method # type: ignore[attr-defined] return memoize_left_rec_wrapper diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py index 9fcca928b08ee5..50cc8520f2ce3f 100644 --- a/Tools/peg_generator/pegen/testutil.py +++ b/Tools/peg_generator/pegen/testutil.py @@ -37,7 +37,7 @@ def generate_parser(grammar: Grammar) -> Type[Parser]: def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = False) -> Any: # Run a parser on a file (stream). - tokenizer = Tokenizer(tokenize.generate_tokens(file.readline)) # type: ignore # typeshed issue #3515 + tokenizer = Tokenizer(tokenize.generate_tokens(file.readline)) # type: ignore[arg-type] # typeshed issue #3515 parser = parser_class(tokenizer, verbose=verbose) result = parser.start() if result is None: @@ -52,7 +52,7 @@ def parse_string( if dedent: source = textwrap.dedent(source) file = io.StringIO(source) - return run_parser(file, parser_class, verbose=verbose) # type: ignore # typeshed issue #3515 + return run_parser(file, parser_class, verbose=verbose) # type: ignore[arg-type] # typeshed issue #3515 def make_parser(source: str) -> Type[Parser]: @@ -116,7 +116,7 @@ def generate_parser_c_extension( def print_memstats() -> bool: MiB: Final = 2**20 try: - import psutil # type: ignore + import psutil # type: ignore[import] except ImportError: return False print("Memory stats:") From e675e515aeed16cb5091e310cb161b10c16a237d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 13:33:28 +0200 Subject: [PATCH 391/598] gh-108494: Argument Clinic: fix option group for Limited C API (#108574) Use PyTuple_Size() instead of PyTuple_GET_SIZE(). --- Tools/clinic/clinic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2b016685e84ce5..ec5bf302416ba8 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1664,7 +1664,8 @@ def group_to_variable_name(group: int) -> str: def render_option_group_parsing( self, f: Function, - template_dict: TemplateDict + template_dict: TemplateDict, + limited_capi: bool, ) -> None: # positional only, grouped, optional arguments! # can be optional on the left or right. @@ -1712,7 +1713,11 @@ def render_option_group_parsing( count_min = sys.maxsize count_max = -1 - add("switch (PyTuple_GET_SIZE(args)) {\n") + if limited_capi: + nargs = 'PyTuple_Size(args)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + add(f"switch ({nargs}) {{\n") for subset in permute_optional_groups(left, required, right): count = len(subset) count_min = min(count_min, count) @@ -1869,7 +1874,8 @@ def render_function( template_dict['unpack_max'] = str(unpack_max) if has_option_groups: - self.render_option_group_parsing(f, template_dict) + self.render_option_group_parsing(f, template_dict, + limited_capi=clinic.limited_capi) # buffers, not destination for name, destination in clinic.destination_buffers.items(): From bf08131e0ae3a2e59a1428c648f433da6921c561 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 13:39:42 +0200 Subject: [PATCH 392/598] gh-108494: Don't build _testclinic_limited with TraceRefs (#108608) If Python is built with ./configure --with-trace-refs, don't build the _testclinic_limited extension. The limited C API (Py_LIMITED_API) is not compatible with Py_TRACE_REFS. --- Modules/Setup.stdlib.in | 2 +- configure | 44 +++++++++++++++++++++++++++++++++++++++++ configure.ac | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 6ed84953dd4216..7b9dc3c5c8e26e 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -161,7 +161,7 @@ @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c -@MODULE__TESTCLINIC_TRUE@_testclinic_limited _testclinic_limited.c +@MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c # Some testing modules MUST be built as shared libraries. *shared* diff --git a/configure b/configure index 15ca703bb9163f..57e3307266c0b1 100755 --- a/configure +++ b/configure @@ -669,6 +669,8 @@ MODULE__TESTBUFFER_FALSE MODULE__TESTBUFFER_TRUE MODULE__TESTINTERNALCAPI_FALSE MODULE__TESTINTERNALCAPI_TRUE +MODULE__TESTCLINIC_LIMITED_FALSE +MODULE__TESTCLINIC_LIMITED_TRUE MODULE__TESTCLINIC_FALSE MODULE__TESTCLINIC_TRUE MODULE__TESTCAPI_FALSE @@ -29955,6 +29957,44 @@ fi printf "%s\n" "$py_cv_module__testclinic" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testclinic_limited" >&5 +printf %s "checking for stdlib extension module _testclinic_limited... " >&6; } + if test "$py_cv_module__testclinic_limited" != "n/a" +then : + + if test "$TEST_MODULES" = yes -a "$with_trace_refs" = "no" +then : + if true +then : + py_cv_module__testclinic_limited=yes +else $as_nop + py_cv_module__testclinic_limited=missing +fi +else $as_nop + py_cv_module__testclinic_limited=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTCLINIC_LIMITED_STATE=$py_cv_module__testclinic_limited$as_nl" + if test "x$py_cv_module__testclinic_limited" = xyes +then : + + + + +fi + if test "$py_cv_module__testclinic_limited" = yes; then + MODULE__TESTCLINIC_LIMITED_TRUE= + MODULE__TESTCLINIC_LIMITED_FALSE='#' +else + MODULE__TESTCLINIC_LIMITED_TRUE='#' + MODULE__TESTCLINIC_LIMITED_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testclinic_limited" >&5 +printf "%s\n" "$py_cv_module__testclinic_limited" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testinternalcapi" >&5 printf %s "checking for stdlib extension module _testinternalcapi... " >&6; } if test "$py_cv_module__testinternalcapi" != "n/a" @@ -30694,6 +30734,10 @@ if test -z "${MODULE__TESTCLINIC_TRUE}" && test -z "${MODULE__TESTCLINIC_FALSE}" as_fn_error $? "conditional \"MODULE__TESTCLINIC\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTCLINIC_LIMITED_TRUE}" && test -z "${MODULE__TESTCLINIC_LIMITED_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTCLINIC_LIMITED\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTINTERNALCAPI_TRUE}" && test -z "${MODULE__TESTINTERNALCAPI_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTINTERNALCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index fab394ec471ce7..6fb6e110647036 100644 --- a/configure.ac +++ b/configure.ac @@ -7229,6 +7229,7 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes -a "$with_trace_refs" = "no"]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) From 30305d6d01e3952f409d352a794e7a367b8c4b8b Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 29 Aug 2023 14:43:11 +0300 Subject: [PATCH 393/598] gh-108623: Fix compile warning in `Modules/_multiprocessing/semaphore.c` (#108624) --- Modules/_multiprocessing/multiprocessing.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index 47257fd5d9fb26..296e0abb29a0f5 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -1,9 +1,14 @@ #ifndef MULTIPROCESSING_H #define MULTIPROCESSING_H +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "structmember.h" #include "pythread.h" +#include "pycore_signal.h" // _PyOS_IsMainThread() /* * Platform includes and definitions From f8be2e262c5c2fdbc9721210ae1cb46edc16db82 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 29 Aug 2023 15:02:52 +0300 Subject: [PATCH 394/598] gh-108455: peg generator: Use `strict_optional=True` for `grammar_parser` (#108629) --- Tools/peg_generator/mypy.ini | 3 --- Tools/peg_generator/pegen/grammar.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 6a45165e8eecb5..3f3db2ea1266c6 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -14,8 +14,5 @@ enable_error_code = truthy-bool,ignore-without-code warn_return_any = False warn_unreachable = False -[mypy-pegen.grammar_parser] -strict_optional = False - [mypy-setuptools.*] ignore_missing_imports = True diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index 03d60d01026f85..fcf868eb1753e5 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -349,7 +349,7 @@ def initial_names(self) -> AbstractSet[str]: Plain = Union[Leaf, Group] Item = Union[Plain, Opt, Repeat, Forced, Lookahead, Rhs, Cut] -RuleName = Tuple[str, str] +RuleName = Tuple[str, Optional[str]] MetaTuple = Tuple[str, Optional[str]] MetaList = List[MetaTuple] RuleList = List[Rule] From 9205dfeca54cb24aa9a984c12a6419d71635be9f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Aug 2023 17:55:56 +0300 Subject: [PATCH 395/598] gh-108635: Make parameters of some implementations of special methods positional-only (GH-108636) --- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 3 - Objects/clinic/typevarobject.c.h | 222 ++---------------- Objects/typevarobject.c | 21 +- PC/clinic/winreg.c.h | 37 +-- PC/winreg.c | 4 +- 8 files changed, 36 insertions(+), 254 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 2f930babf6aad4..d2b6058362ca23 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -789,7 +789,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 5a0cd1a02ba561..169528d6eb1fee 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -278,7 +278,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(aggregate_class) STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) - STRUCT_FOR_ID(arg) STRUCT_FOR_ID(argdefs) STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 694a409fbf4f47..ccfedf9b32d2a0 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -787,7 +787,6 @@ extern "C" { INIT_ID(aggregate_class), \ INIT_ID(alias), \ INIT_ID(append), \ - INIT_ID(arg), \ INIT_ID(argdefs), \ INIT_ID(args), \ INIT_ID(arguments), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 841eb787011271..ff7aeba1de96bf 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -675,9 +675,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(arg); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 54189b98446814..5d12be5361cd2f 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -110,58 +110,12 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(typevar_typing_subst__doc__, -"__typing_subst__($self, /, arg)\n" +"__typing_subst__($self, arg, /)\n" "--\n" "\n"); #define TYPEVAR_TYPING_SUBST_METHODDEF \ - {"__typing_subst__", _PyCFunction_CAST(typevar_typing_subst), METH_FASTCALL|METH_KEYWORDS, typevar_typing_subst__doc__}, - -static PyObject * -typevar_typing_subst_impl(typevarobject *self, PyObject *arg); - -static PyObject * -typevar_typing_subst(typevarobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(arg), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"arg", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "__typing_subst__", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *arg; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - arg = args[0]; - return_value = typevar_typing_subst_impl(self, arg); - -exit: - return return_value; -} + {"__typing_subst__", (PyCFunction)typevar_typing_subst, METH_O, typevar_typing_subst__doc__}, PyDoc_STRVAR(typevar_reduce__doc__, "__reduce__($self, /)\n" @@ -386,106 +340,33 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(paramspec_typing_subst__doc__, -"__typing_subst__($self, /, arg)\n" +"__typing_subst__($self, arg, /)\n" "--\n" "\n"); #define PARAMSPEC_TYPING_SUBST_METHODDEF \ - {"__typing_subst__", _PyCFunction_CAST(paramspec_typing_subst), METH_FASTCALL|METH_KEYWORDS, paramspec_typing_subst__doc__}, - -static PyObject * -paramspec_typing_subst_impl(paramspecobject *self, PyObject *arg); - -static PyObject * -paramspec_typing_subst(paramspecobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(arg), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"arg", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "__typing_subst__", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *arg; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - arg = args[0]; - return_value = paramspec_typing_subst_impl(self, arg); - -exit: - return return_value; -} + {"__typing_subst__", (PyCFunction)paramspec_typing_subst, METH_O, paramspec_typing_subst__doc__}, PyDoc_STRVAR(paramspec_typing_prepare_subst__doc__, -"__typing_prepare_subst__($self, /, alias, args)\n" +"__typing_prepare_subst__($self, alias, args, /)\n" "--\n" "\n"); #define PARAMSPEC_TYPING_PREPARE_SUBST_METHODDEF \ - {"__typing_prepare_subst__", _PyCFunction_CAST(paramspec_typing_prepare_subst), METH_FASTCALL|METH_KEYWORDS, paramspec_typing_prepare_subst__doc__}, + {"__typing_prepare_subst__", _PyCFunction_CAST(paramspec_typing_prepare_subst), METH_FASTCALL, paramspec_typing_prepare_subst__doc__}, static PyObject * paramspec_typing_prepare_subst_impl(paramspecobject *self, PyObject *alias, PyObject *args); static PyObject * -paramspec_typing_prepare_subst(paramspecobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +paramspec_typing_prepare_subst(paramspecobject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(alias), &_Py_ID(args), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"alias", "args", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "__typing_prepare_subst__", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; PyObject *alias; PyObject *__clinic_args; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { goto exit; } alias = args[0]; @@ -572,106 +453,33 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(typevartuple_typing_subst__doc__, -"__typing_subst__($self, /, arg)\n" +"__typing_subst__($self, arg, /)\n" "--\n" "\n"); #define TYPEVARTUPLE_TYPING_SUBST_METHODDEF \ - {"__typing_subst__", _PyCFunction_CAST(typevartuple_typing_subst), METH_FASTCALL|METH_KEYWORDS, typevartuple_typing_subst__doc__}, - -static PyObject * -typevartuple_typing_subst_impl(typevartupleobject *self, PyObject *arg); - -static PyObject * -typevartuple_typing_subst(typevartupleobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(arg), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"arg", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "__typing_subst__", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *arg; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - arg = args[0]; - return_value = typevartuple_typing_subst_impl(self, arg); - -exit: - return return_value; -} + {"__typing_subst__", (PyCFunction)typevartuple_typing_subst, METH_O, typevartuple_typing_subst__doc__}, PyDoc_STRVAR(typevartuple_typing_prepare_subst__doc__, -"__typing_prepare_subst__($self, /, alias, args)\n" +"__typing_prepare_subst__($self, alias, args, /)\n" "--\n" "\n"); #define TYPEVARTUPLE_TYPING_PREPARE_SUBST_METHODDEF \ - {"__typing_prepare_subst__", _PyCFunction_CAST(typevartuple_typing_prepare_subst), METH_FASTCALL|METH_KEYWORDS, typevartuple_typing_prepare_subst__doc__}, + {"__typing_prepare_subst__", _PyCFunction_CAST(typevartuple_typing_prepare_subst), METH_FASTCALL, typevartuple_typing_prepare_subst__doc__}, static PyObject * typevartuple_typing_prepare_subst_impl(typevartupleobject *self, PyObject *alias, PyObject *args); static PyObject * -typevartuple_typing_prepare_subst(typevartupleobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +typevartuple_typing_prepare_subst(typevartupleobject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(alias), &_Py_ID(args), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"alias", "args", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "__typing_prepare_subst__", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; PyObject *alias; PyObject *__clinic_args; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { goto exit; } alias = args[0]; @@ -783,4 +591,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=807bcd30ebd10ac3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d614edf64f28e346 input=a9049054013a1b77]*/ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index ef1803d0fd5526..3adae3e14c9103 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -402,12 +402,13 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, typevar.__typing_subst__ as typevar_typing_subst arg: object + / [clinic start generated code]*/ static PyObject * -typevar_typing_subst_impl(typevarobject *self, PyObject *arg) -/*[clinic end generated code: output=c76ced134ed8f4e1 input=6b70a4bb2da838de]*/ +typevar_typing_subst(typevarobject *self, PyObject *arg) +/*[clinic end generated code: output=0773735e8ce18968 input=9e87b57f0fc59b92]*/ { PyObject *args[2] = {(PyObject *)self, arg}; PyObject *result = call_typing_func_object("_typevar_subst", args, 2); @@ -884,12 +885,13 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, paramspec.__typing_subst__ as paramspec_typing_subst arg: object + / [clinic start generated code]*/ static PyObject * -paramspec_typing_subst_impl(paramspecobject *self, PyObject *arg) -/*[clinic end generated code: output=803e1ade3f13b57d input=4e0005d24023e896]*/ +paramspec_typing_subst(paramspecobject *self, PyObject *arg) +/*[clinic end generated code: output=4c5b4aaada1c5814 input=2d5b5e3d4a717189]*/ { PyObject *args[2] = {(PyObject *)self, arg}; PyObject *result = call_typing_func_object("_paramspec_subst", args, 2); @@ -901,13 +903,14 @@ paramspec.__typing_prepare_subst__ as paramspec_typing_prepare_subst alias: object args: object + / [clinic start generated code]*/ static PyObject * paramspec_typing_prepare_subst_impl(paramspecobject *self, PyObject *alias, PyObject *args) -/*[clinic end generated code: output=95449d630a2adb9a input=4375e2ffcb2ad635]*/ +/*[clinic end generated code: output=95449d630a2adb9a input=6df6f9fef3e150da]*/ { PyObject *args_array[3] = {(PyObject *)self, alias, args}; PyObject *result = call_typing_func_object( @@ -1106,12 +1109,13 @@ typevartuple_impl(PyTypeObject *type, PyObject *name) typevartuple.__typing_subst__ as typevartuple_typing_subst arg: object + / [clinic start generated code]*/ static PyObject * -typevartuple_typing_subst_impl(typevartupleobject *self, PyObject *arg) -/*[clinic end generated code: output=814316519441cd76 input=670c4e0a36e5d8c0]*/ +typevartuple_typing_subst(typevartupleobject *self, PyObject *arg) +/*[clinic end generated code: output=237054c6d7484eea input=3fcf2dfd9eee7945]*/ { PyErr_SetString(PyExc_TypeError, "Substitution of bare TypeVarTuple is not supported"); return NULL; @@ -1122,13 +1126,14 @@ typevartuple.__typing_prepare_subst__ as typevartuple_typing_prepare_subst alias: object args: object + / [clinic start generated code]*/ static PyObject * typevartuple_typing_prepare_subst_impl(typevartupleobject *self, PyObject *alias, PyObject *args) -/*[clinic end generated code: output=ff999bc5b02036c1 input=a211b05f2eeb4306]*/ +/*[clinic end generated code: output=ff999bc5b02036c1 input=685b149b0fc47556]*/ { PyObject *args_array[3] = {(PyObject *)self, alias, args}; PyObject *result = call_typing_func_object( diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 8cb8749498e3df..6e23929c185caf 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -93,53 +93,26 @@ winreg_HKEYType___enter__(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) #if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) PyDoc_STRVAR(winreg_HKEYType___exit____doc__, -"__exit__($self, /, exc_type, exc_value, traceback)\n" +"__exit__($self, exc_type, exc_value, traceback, /)\n" "--\n" "\n"); #define WINREG_HKEYTYPE___EXIT___METHODDEF \ - {"__exit__", _PyCFunction_CAST(winreg_HKEYType___exit__), METH_FASTCALL|METH_KEYWORDS, winreg_HKEYType___exit____doc__}, + {"__exit__", _PyCFunction_CAST(winreg_HKEYType___exit__), METH_FASTCALL, winreg_HKEYType___exit____doc__}, static PyObject * winreg_HKEYType___exit___impl(PyHKEYObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *traceback); static PyObject * -winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(exc_type), &_Py_ID(exc_value), &_Py_ID(traceback), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"exc_type", "exc_value", "traceback", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "__exit__", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; PyObject *exc_type; PyObject *exc_value; PyObject *traceback; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { goto exit; } exc_type = args[0]; @@ -1789,4 +1762,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=00343ee8da923da8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2431b1b06b148722 input=a9049054013a1b77]*/ diff --git a/PC/winreg.c b/PC/winreg.c index ee47a3fd40fc86..767127d34cab1b 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -324,12 +324,14 @@ winreg.HKEYType.__exit__ exc_type: object exc_value: object traceback: object + / + [clinic start generated code]*/ static PyObject * winreg_HKEYType___exit___impl(PyHKEYObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *traceback) -/*[clinic end generated code: output=923ebe7389e6a263 input=fb32489ee92403c7]*/ +/*[clinic end generated code: output=923ebe7389e6a263 input=1eac83cd06962689]*/ { winreg_state *st = _PyType_GetModuleState(Py_TYPE(self)); assert(st != NULL); From 7659128b9d7a30ddbcb063bc12e2ddb0f1f119e0 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 29 Aug 2023 17:00:43 +0200 Subject: [PATCH 396/598] Use non alternate name for Kyiv (GH-108533) tzdata provides Kiev as an alternative to Kyiv: https://sources.debian.org/src/tzdata/2023c-10/backward/?hl=314#L314 But Debian moved it to the tzdata-legacy package breaking the test: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1050530 This patch switches to the name provided by tzdata. --- Lib/test/test_email/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index 25fa48c5ee217b..5768c7d0eb57eb 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -148,7 +148,7 @@ def test_localtime_epoch_notz_daylight_false(self): @unittest.skipUnless(os.path.exists('/usr/share/zoneinfo') or os.path.exists('/usr/lib/zoneinfo'), "Can't find the Olson's TZ database") - @test.support.run_with_tz('Europe/Kiev') + @test.support.run_with_tz('Europe/Kyiv') def test_variable_tzname(self): t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) From b62a76043e543fbb15cab7e9e8874a798bc600d2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 17:46:46 +0200 Subject: [PATCH 397/598] gh-108638: Fix stat.filemode() when _stat is missing (#108639) Change the pure Python implementation of stat.filemode() for unknown file type: use "?", as done by the _stat.filemode(). test_stat skips TestFilemodeCStat if the _stat extension is missing. --- Lib/stat.py | 10 ++++++++-- Lib/test/test_stat.py | 6 +++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Lib/stat.py b/Lib/stat.py index fc024db3f4fbee..52cadbf04f6c88 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -126,6 +126,8 @@ def S_ISWHT(mode): _filemode_table = ( + # File type chars according to: + # http://en.wikibooks.org/wiki/C_Programming/POSIX_Reference/sys/stat.h ((S_IFLNK, "l"), (S_IFSOCK, "s"), # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR (S_IFREG, "-"), @@ -156,13 +158,17 @@ def S_ISWHT(mode): def filemode(mode): """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" perm = [] - for table in _filemode_table: + for index, table in enumerate(_filemode_table): for bit, char in table: if mode & bit == bit: perm.append(char) break else: - perm.append("-") + if index == 0: + # Unknown filetype + perm.append("?") + else: + perm.append("-") return "".join(perm) diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 4ba37aed2dc9db..0eced2fcf98376 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -122,8 +122,11 @@ def test_mode(self): st_mode, modestr = self.get_mode() self.assertEqual(modestr, '-rwx------') self.assertS_IS("REG", st_mode) - self.assertEqual(self.statmod.S_IMODE(st_mode), + imode = self.statmod.S_IMODE(st_mode) + self.assertEqual(imode, self.statmod.S_IRWXU) + self.assertEqual(self.statmod.filemode(imode), + '?rwx------') os.chmod(TESTFN, 0o070) st_mode, modestr = self.get_mode() @@ -238,6 +241,7 @@ def test_file_attribute_constants(self): self.assertEqual(value, modvalue, key) +@unittest.skipIf(c_stat is None, 'need _stat extension') class TestFilemodeCStat(TestFilemode, unittest.TestCase): statmod = c_stat From 83e191ba7627cbcb06cebcc0a020b847f271ca59 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Aug 2023 18:04:07 +0200 Subject: [PATCH 398/598] test_sys: remove debug print() (#108642) --- Lib/test/test_sys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9dce15ed1529e7..f3608ce142fb9b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -732,7 +732,6 @@ def test_subinterp_intern_statically_allocated(self): s = '__init__' t = sys.intern(s) - print('------------------------') interp = interpreters.create() interp.run(textwrap.dedent(f''' import sys From 4f22152713d008cdd7c1d373a0f0c8dcf30e217e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 29 Aug 2023 09:51:51 -0700 Subject: [PATCH 399/598] gh-107557: Remove unnecessary SAVE_IP instructions (#108583) Also remove NOP instructions. The "stubs" are not optimized in this fashion (their SAVE_IP should always be preserved since it's where to jump next, and they don't contain NOPs by their nature). --- Include/internal/pycore_opcode_metadata.h | 450 ++++++++++++---------- Python/optimizer.c | 140 +++++-- Tools/cases_generator/analysis.py | 2 +- Tools/cases_generator/flags.py | 26 +- Tools/cases_generator/generate_cases.py | 5 +- 5 files changed, 392 insertions(+), 231 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 49089721c6ac20..aaf3541e095468 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -37,38 +37,42 @@ #define _BINARY_OP_SUBTRACT_FLOAT 309 #define _GUARD_BOTH_UNICODE 310 #define _BINARY_OP_ADD_UNICODE 311 -#define _POP_FRAME 312 -#define _LOAD_LOCALS 313 -#define _LOAD_FROM_DICT_OR_GLOBALS 314 -#define _GUARD_GLOBALS_VERSION 315 -#define _GUARD_BUILTINS_VERSION 316 -#define _LOAD_GLOBAL_MODULE 317 -#define _LOAD_GLOBAL_BUILTINS 318 -#define _GUARD_TYPE_VERSION 319 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 -#define _LOAD_ATTR_INSTANCE_VALUE 321 -#define IS_NONE 322 -#define _ITER_CHECK_LIST 323 -#define _IS_ITER_EXHAUSTED_LIST 324 -#define _ITER_NEXT_LIST 325 -#define _ITER_CHECK_TUPLE 326 -#define _IS_ITER_EXHAUSTED_TUPLE 327 -#define _ITER_NEXT_TUPLE 328 -#define _ITER_CHECK_RANGE 329 -#define _IS_ITER_EXHAUSTED_RANGE 330 -#define _ITER_NEXT_RANGE 331 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 332 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 333 -#define _CHECK_PEP_523 334 -#define _CHECK_FUNCTION_EXACT_ARGS 335 -#define _CHECK_STACK_SPACE 336 -#define _INIT_CALL_PY_EXACT_ARGS 337 -#define _PUSH_FRAME 338 -#define _POP_JUMP_IF_FALSE 339 -#define _POP_JUMP_IF_TRUE 340 -#define JUMP_TO_TOP 341 -#define SAVE_CURRENT_IP 342 -#define INSERT 343 +#define _BINARY_OP_INPLACE_ADD_UNICODE 312 +#define _POP_FRAME 313 +#define _LOAD_LOCALS 314 +#define _LOAD_FROM_DICT_OR_GLOBALS 315 +#define _GUARD_GLOBALS_VERSION 316 +#define _GUARD_BUILTINS_VERSION 317 +#define _LOAD_GLOBAL_MODULE 318 +#define _LOAD_GLOBAL_BUILTINS 319 +#define _GUARD_TYPE_VERSION 320 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 321 +#define _LOAD_ATTR_INSTANCE_VALUE 322 +#define IS_NONE 323 +#define _ITER_CHECK_LIST 324 +#define _ITER_JUMP_LIST 325 +#define _IS_ITER_EXHAUSTED_LIST 326 +#define _ITER_NEXT_LIST 327 +#define _ITER_CHECK_TUPLE 328 +#define _ITER_JUMP_TUPLE 329 +#define _IS_ITER_EXHAUSTED_TUPLE 330 +#define _ITER_NEXT_TUPLE 331 +#define _ITER_CHECK_RANGE 332 +#define _ITER_JUMP_RANGE 333 +#define _IS_ITER_EXHAUSTED_RANGE 334 +#define _ITER_NEXT_RANGE 335 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 336 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 337 +#define _CHECK_PEP_523 338 +#define _CHECK_FUNCTION_EXACT_ARGS 339 +#define _CHECK_STACK_SPACE 340 +#define _INIT_CALL_PY_EXACT_ARGS 341 +#define _PUSH_FRAME 342 +#define _POP_JUMP_IF_FALSE 343 +#define _POP_JUMP_IF_TRUE 344 +#define JUMP_TO_TOP 345 +#define SAVE_CURRENT_IP 346 +#define INSERT 347 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -1159,6 +1163,8 @@ enum InstructionFormat { #define HAS_FREE_FLAG (16) #define HAS_LOCAL_FLAG (32) #define HAS_EVAL_BREAK_FLAG (64) +#define HAS_DEOPT_FLAG (128) +#define HAS_ERROR_FLAG (256) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1166,6 +1172,8 @@ enum InstructionFormat { #define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG)) #define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) #define OPCODE_HAS_EVAL_BREAK(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EVAL_BREAK_FLAG)) +#define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) +#define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) struct opcode_metadata { bool valid_entry; @@ -1198,10 +1206,10 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SI #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1213,198 +1221,239 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [END_FOR] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, 0 }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, - [TO_BOOL] = { true, INSTR_FMT_IXC00, 0 }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, 0 }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, 0 }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, 0 }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, 0 }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, 0 }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, 0 }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, 0 }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG }, - [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, - [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, - [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, 0 }, - [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, 0 }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, 0 }, - [DELETE_SUBSCR] = { true, INSTR_FMT_IX, 0 }, - [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, 0 }, + [_POP_FRAME] = { true, INSTR_FMT_IX, 0 }, [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [GET_AITER] = { true, INSTR_FMT_IX, 0 }, - [GET_ANEXT] = { true, INSTR_FMT_IX, 0 }, - [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG }, + [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, 0 }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, 0 }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, 0 }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, - [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, 0 }, - [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, 0 }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [_LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [_LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_GLOBAL_BUILTINS] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, 0 }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_METHOD] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, 0 }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, 0 }, - [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_ZERO_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_ZERO_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_METHOD] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, + [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, 0 }, - [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, 0 }, - [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [JUMP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [IS_NONE] = { true, INSTR_FMT_IX, 0 }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [GET_LEN] = { true, INSTR_FMT_IX, 0 }, - [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [MATCH_KEYS] = { true, INSTR_FMT_IX, 0 }, - [GET_ITER] = { true, INSTR_FMT_IX, 0 }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, 0 }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, 0 }, - [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, 0 }, + [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_IS_ITER_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, 0 }, + [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_IS_ITER_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, 0 }, + [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_IS_ITER_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, 0 }, + [_ITER_NEXT_RANGE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [SETUP_FINALLY] = { true, INSTR_FMT_IX, 0 }, [SETUP_CLEANUP] = { true, INSTR_FMT_IX, 0 }, [SETUP_WITH] = { true, INSTR_FMT_IX, 0 }, [POP_BLOCK] = { true, INSTR_FMT_IX, 0 }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [KW_NAMES] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, 0 }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_PUSH_FRAME] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, 0 }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, 0 }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, 0 }, - [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, 0 }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1414,6 +1463,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, + [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, + [SAVE_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [SAVE_CURRENT_IP] = { true, INSTR_FMT_IX, 0 }, + [EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, + [INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, }; #endif // NEED_OPCODE_METADATA @@ -1570,6 +1626,7 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_POP_FRAME] = "_POP_FRAME", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", @@ -1582,12 +1639,15 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", [IS_NONE] = "IS_NONE", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", [_IS_ITER_EXHAUSTED_LIST] = "_IS_ITER_EXHAUSTED_LIST", [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_JUMP_TUPLE] = "_ITER_JUMP_TUPLE", [_IS_ITER_EXHAUSTED_TUPLE] = "_IS_ITER_EXHAUSTED_TUPLE", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_JUMP_RANGE] = "_ITER_JUMP_RANGE", [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", diff --git a/Python/optimizer.c b/Python/optimizer.c index bbc125954c701e..c311a03fa59a2f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -372,6 +372,32 @@ static PyTypeObject UOpExecutor_Type = { .tp_as_sequence = &uop_as_sequence, }; +static int +move_stubs( + _PyUOpInstruction *trace, + int trace_length, + int stubs_start, + int stubs_end +) +{ + memmove(trace + trace_length, + trace + stubs_start, + (stubs_end - stubs_start) * sizeof(_PyUOpInstruction)); + // Patch up the jump targets + for (int i = 0; i < trace_length; i++) { + if (trace[i].opcode == _POP_JUMP_IF_FALSE || + trace[i].opcode == _POP_JUMP_IF_TRUE) + { + int target = trace[i].oparg; + if (target >= stubs_start) { + target += trace_length - stubs_start; + trace[i].oparg = target; + } + } + } + return trace_length + stubs_end - stubs_start; +} + #define TRACE_STACK_SIZE 5 static int @@ -730,36 +756,31 @@ translate_bytecode_to_trace( if (trace_length > 3) { ADD_TO_TRACE(EXIT_TRACE, 0, 0); DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", + "Created a trace for %s (%s:%d) at byte offset %d -- length %d+%d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code), - trace_length); - if (max_length < buffer_size && trace_length < max_length) { - // Move the stubs back to be immediately after the main trace - // (which ends at trace_length) - DPRINTF(2, - "Moving %d stub uops back by %d\n", - buffer_size - max_length, - max_length - trace_length); - memmove(trace + trace_length, - trace + max_length, - (buffer_size - max_length) * sizeof(_PyUOpInstruction)); - // Patch up the jump targets - for (int i = 0; i < trace_length; i++) { - if (trace[i].opcode == _POP_JUMP_IF_FALSE || - trace[i].opcode == _POP_JUMP_IF_TRUE) - { - int target = trace[i].oparg; - if (target >= max_length) { - target += trace_length - max_length; - trace[i].oparg = target; - } - } + trace_length, + buffer_size - max_length); + if (max_length < buffer_size) { + // There are stubs + if (trace_length < max_length) { + // There's a gap before the stubs + // Move the stubs back to be immediately after the main trace + // (which ends at trace_length) + DPRINTF(2, + "Moving %d stub uops back by %d\n", + buffer_size - max_length, + max_length - trace_length); + trace_length = move_stubs(trace, trace_length, max_length, buffer_size); + } + else { + assert(trace_length == max_length); + // There's no gap + trace_length = buffer_size; } } - trace_length += buffer_size - max_length; return trace_length; } else { @@ -779,6 +800,76 @@ translate_bytecode_to_trace( #undef DPRINTF } +static int +remove_unneeded_uops(_PyUOpInstruction *trace, int trace_length) +{ + // Stage 1: Replace unneeded SAVE_IP uops with NOP. + // Note that we don't enter stubs, those SAVE_IPs are needed. + int last_save_ip = -1; + int last_instr = 0; + bool need_ip = true; + for (int pc = 0; pc < trace_length; pc++) { + int opcode = trace[pc].opcode; + if (opcode == SAVE_CURRENT_IP) { + // Special case: never remove preceding SAVE_IP + last_save_ip = -1; + } + else if (opcode == SAVE_IP) { + if (!need_ip && last_save_ip >= 0) { + trace[last_save_ip].opcode = NOP; + } + need_ip = false; + last_save_ip = pc; + } + else if (opcode == JUMP_TO_TOP || opcode == EXIT_TRACE) { + last_instr = pc + 1; + break; + } + else { + // If opcode has ERROR or DEOPT, set need_up to true + if (_PyOpcode_opcode_metadata[opcode].flags & (HAS_ERROR_FLAG | HAS_DEOPT_FLAG)) { + need_ip = true; + } + } + } + // Stage 2: Squash NOP opcodes (pre-existing or set above). + int dest = 0; + for (int pc = 0; pc < last_instr; pc++) { + int opcode = trace[pc].opcode; + if (opcode != NOP) { + if (pc != dest) { + trace[dest] = trace[pc]; + } + dest++; + } + } + // Stage 3: Move the stubs back. + if (dest < last_instr) { + int new_trace_length = move_stubs(trace, dest, last_instr, trace_length); +#ifdef Py_DEBUG + char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + if (lltrace >= 2) { + printf("Optimized trace (length %d+%d = %d, saved %d):\n", + dest, trace_length - last_instr, new_trace_length, + trace_length - new_trace_length); + for (int pc = 0; pc < new_trace_length; pc++) { + printf("%4d: (%s, %d, %" PRIu64 ")\n", + pc, + uop_name(trace[pc].opcode), + (trace[pc].oparg), + (uint64_t)(trace[pc].operand)); + } + } +#endif + trace_length = new_trace_length; + } + return trace_length; +} + static int uop_optimize( _PyOptimizerObject *self, @@ -798,6 +889,7 @@ uop_optimize( if (uop_optimize != NULL && *uop_optimize > '0') { trace_length = _Py_uop_analyze_and_optimize(code, trace, trace_length, curr_stackentries); } + trace_length = remove_unneeded_uops(trace, trace_length); _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); if (executor == NULL) { return -1; diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index b3c3df1378b943..7c7c9086cd72f5 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -381,7 +381,7 @@ def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: # Make sure the targets have the same fmt fmts = list(set([t.instr_fmt for t in targets])) assert len(fmts) == 1 - ignored_flags = {'HAS_EVAL_BREAK_FLAG'} + ignored_flags = {"HAS_EVAL_BREAK_FLAG", "HAS_DEOPT_FLAG", "HAS_ERROR_FLAG"} assert len({t.instr_flags.bitmap(ignore=ignored_flags) for t in targets}) == 1 return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 193e7ff0d9ccfb..5241331bb97cdb 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -10,13 +10,15 @@ class InstructionFlags: """Construct and manipulate instruction flags""" - HAS_ARG_FLAG: bool - HAS_CONST_FLAG: bool - HAS_NAME_FLAG: bool - HAS_JUMP_FLAG: bool - HAS_FREE_FLAG: bool - HAS_LOCAL_FLAG: bool - HAS_EVAL_BREAK_FLAG: bool + HAS_ARG_FLAG: bool = False + HAS_CONST_FLAG: bool = False + HAS_NAME_FLAG: bool = False + HAS_JUMP_FLAG: bool = False + HAS_FREE_FLAG: bool = False + HAS_LOCAL_FLAG: bool = False + HAS_EVAL_BREAK_FLAG: bool = False + HAS_DEOPT_FLAG: bool = False + HAS_ERROR_FLAG: bool = False def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @@ -40,11 +42,19 @@ def fromInstruction(instr: parsing.Node) -> "InstructionFlags": ) and not has_free, HAS_EVAL_BREAK_FLAG=variable_used(instr, "CHECK_EVAL_BREAKER"), + HAS_DEOPT_FLAG=variable_used(instr, "DEOPT_IF"), + HAS_ERROR_FLAG=( + variable_used(instr, "ERROR_IF") + or variable_used(instr, "error") + or variable_used(instr, "pop_1_error") + or variable_used(instr, "exception_unwind") + or variable_used(instr, "resume_with_error") + ), ) @staticmethod def newEmpty() -> "InstructionFlags": - return InstructionFlags(False, False, False, False, False, False, False) + return InstructionFlags() def add(self, other: "InstructionFlags") -> None: for name, value in dataclasses.asdict(other).items(): diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f89572fc496a70..1335c0c29ebbf6 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -484,8 +484,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case OverriddenInstructionPlaceHolder(): continue case parsing.InstDef(): - if thing.kind != "op": - self.write_metadata_for_inst(self.instrs[thing.name]) + self.write_metadata_for_inst(self.instrs[thing.name]) case parsing.Macro(): self.write_metadata_for_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): @@ -653,7 +652,7 @@ def add(name: str) -> None: add("SAVE_IP") for instr in self.instrs.values(): - if instr.kind == "op" and instr.is_viable_uop(): + if instr.kind == "op": add(instr.name) def write_macro_expansions( From 59e46932c8d2dc6fe84a8cf144dde962838c0204 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 29 Aug 2023 11:14:56 -0700 Subject: [PATCH 400/598] gh-108488: Initialize JUMP_BACKWARD cache to 0, not 17 (#108591) This mis-initialization caused the executor optimization to kick in sooner than intended. It also set the lower 4 bits of the counter to `1` -- those bits are supposed to be reserved (the actual counter is in the upper 12 bits). --- Lib/test/test_capi/test_misc.py | 4 ++-- .../2023-08-28-22-22-15.gh-issue-108488.e8-fxg.rst | 1 + Python/specialize.c | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-28-22-22-15.gh-issue-108488.e8-fxg.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 1cd4c56b49bba5..4148f15b2aa662 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2478,7 +2478,7 @@ def testfunc(a): opt = _testinternalcapi.get_uop_optimizer() with temporary_optimizer(opt): - testfunc([1, 2, 3]) + testfunc(range(10)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -2493,7 +2493,7 @@ def testfunc(a): opt = _testinternalcapi.get_uop_optimizer() with temporary_optimizer(opt): - testfunc([1, 2, 3]) + testfunc(range(10)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-28-22-22-15.gh-issue-108488.e8-fxg.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-28-22-22-15.gh-issue-108488.e8-fxg.rst new file mode 100644 index 00000000000000..f9d6f593b8eb9b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-28-22-22-15.gh-issue-108488.e8-fxg.rst @@ -0,0 +1 @@ +Change the initialization of inline cache entries so that the cache entry for ``JUMP_BACKWARD`` is initialized to zero, instead of the ``adaptive_counter_warmup()`` value used for all other instructions. This counter, unique among instructions, counts up from zero. diff --git a/Python/specialize.c b/Python/specialize.c index a467f163f2ca9d..a794f146c188ec 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -302,7 +302,9 @@ _PyCode_Quicken(PyCodeObject *code) assert(opcode < MIN_INSTRUMENTED_OPCODE); int caches = _PyOpcode_Caches[opcode]; if (caches) { - instructions[i + 1].cache = adaptive_counter_warmup(); + // JUMP_BACKWARD counter counts up from 0 until it is > backedge_threshold + instructions[i + 1].cache = + opcode == JUMP_BACKWARD ? 0 : adaptive_counter_warmup(); i += caches; } } From 8178a88bd81edae87d6974483e4de9b32e808797 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 29 Aug 2023 21:13:38 +0200 Subject: [PATCH 401/598] gh-107801: Improve the accuracy of io.IOBase.seek docs (#108268) - Add param docstrings - Link to os.SEEK_* constants - Mention the return value in the initial paragraph Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/io.rst | 26 ++++++++++++++------------ Modules/_io/clinic/iobase.c.h | 7 ++++++- Modules/_io/iobase.c | 4 +++- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 25f4d0d5841333..01088879218cb4 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -403,20 +403,19 @@ I/O Base Classes Note that it's already possible to iterate on file objects using ``for line in file: ...`` without calling :meth:`!file.readlines`. - .. method:: seek(offset, whence=SEEK_SET, /) + .. method:: seek(offset, whence=os.SEEK_SET, /) - Change the stream position to the given byte *offset*. *offset* is - interpreted relative to the position indicated by *whence*. The default - value for *whence* is :data:`!SEEK_SET`. Values for *whence* are: + Change the stream position to the given byte *offset*, + interpreted relative to the position indicated by *whence*, + and return the new absolute position. + Values for *whence* are: - * :data:`!SEEK_SET` or ``0`` -- start of the stream (the default); + * :data:`os.SEEK_SET` or ``0`` -- start of the stream (the default); *offset* should be zero or positive - * :data:`!SEEK_CUR` or ``1`` -- current stream position; *offset* may - be negative - * :data:`!SEEK_END` or ``2`` -- end of the stream; *offset* is usually - negative - - Return the new absolute position. + * :data:`os.SEEK_CUR` or ``1`` -- current stream position; + *offset* may be negative + * :data:`os.SEEK_END` or ``2`` -- end of the stream; + *offset* is usually negative .. versionadded:: 3.1 The :data:`!SEEK_*` constants. @@ -1061,6 +1060,10 @@ Text I/O Any other argument combinations are invalid, and may raise exceptions. + .. seealso:: + + :data:`os.SEEK_SET`, :data:`os.SEEK_CUR`, and :data:`os.SEEK_END`. + .. method:: tell() Return the stream position as an opaque number. @@ -1068,7 +1071,6 @@ Text I/O to restore a previous stream position. - .. class:: StringIO(initial_value='', newline='\n') A text stream using an in-memory text buffer. It inherits diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index f582b9d2c71042..fb96ea132f3dc5 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -15,6 +15,11 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, "\n" "Change the stream position to the given byte offset.\n" "\n" +" offset\n" +" The stream position, relative to \'whence\'.\n" +" whence\n" +" The relative position to seek from.\n" +"\n" "The offset is interpreted relative to the position indicated by whence.\n" "Values for whence are:\n" "\n" @@ -437,4 +442,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=ec741e0961671a86 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d96f5bfd72e6eafb input=a9049054013a1b77]*/ diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 5fd19895311c0c..55508a2214f84d 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -83,7 +83,9 @@ iobase_unsupported(_PyIO_State *state, const char *message) _io._IOBase.seek cls: defining_class offset: int(unused=True) + The stream position, relative to 'whence'. whence: int(unused=True, c_default='0') = os.SEEK_SET + The relative position to seek from. / Change the stream position to the given byte offset. @@ -101,7 +103,7 @@ Return the new absolute position. static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(offset), int Py_UNUSED(whence)) -/*[clinic end generated code: output=8bd74ea6538ded53 input=8d4e6adcd08292f2]*/ +/*[clinic end generated code: output=8bd74ea6538ded53 input=74211232b363363e]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); From 77e8f233acd39420dc8921960715bf6b29587fee Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 29 Aug 2023 20:14:08 +0100 Subject: [PATCH 402/598] gh-108455: peg_generator: install two stubs packages before running mypy (#108637) --- Tools/peg_generator/mypy.ini | 6 ++++-- Tools/peg_generator/pegen/build.py | 11 ++++++++--- Tools/peg_generator/pegen/testutil.py | 2 +- Tools/requirements-dev.txt | 4 ++++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 3f3db2ea1266c6..55429dd76eba68 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -14,5 +14,7 @@ enable_error_code = truthy-bool,ignore-without-code warn_return_any = False warn_unreachable = False -[mypy-setuptools.*] -ignore_missing_imports = True +[mypy-pegen.build] +# we need this for now due to some missing annotations +# in typeshed's stubs for setuptools +disallow_untyped_calls = False diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 81efcadc001ec5..998cd41428db19 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,4 +1,5 @@ import itertools +import logging import os import pathlib import sys @@ -90,6 +91,7 @@ def compile_c_extension( static library of the common parser sources (this is useful in case you are creating multiple extensions). """ + import setuptools.command.build_ext import setuptools.logging from setuptools import Extension, Distribution @@ -98,7 +100,7 @@ def compile_c_extension( from setuptools._distutils.sysconfig import customize_compiler if verbose: - setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG) + setuptools.logging.set_threshold(logging.DEBUG) source_file_path = pathlib.Path(generated_source_path) extension_name = source_file_path.stem @@ -140,12 +142,14 @@ def compile_c_extension( ) dist = Distribution({"name": extension_name, "ext_modules": [extension]}) cmd = dist.get_command_obj("build_ext") + assert isinstance(cmd, setuptools.command.build_ext.build_ext) fixup_build_ext(cmd) cmd.build_lib = str(source_file_path.parent) cmd.include_dirs = include_dirs if build_dir: cmd.build_temp = build_dir - cmd.ensure_finalized() + # A deficiency in typeshed's stubs means we have to type: ignore: + cmd.ensure_finalized() # type: ignore[attr-defined] compiler = new_compiler() customize_compiler(compiler) @@ -156,7 +160,8 @@ def compile_c_extension( library_filename = compiler.library_filename(extension_name, output_dir=library_dir) if newer_group(common_sources, library_filename, "newer"): if sys.platform == "win32": - pdb = compiler.static_lib_format % (extension_name, ".pdb") + # A deficiency in typeshed's stubs means we have to type: ignore: + pdb = compiler.static_lib_format % (extension_name, ".pdb") # type: ignore[attr-defined] compile_opts = [f"/Fd{library_dir}\\{pdb}"] compile_opts.extend(extra_compile_args) else: diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py index 50cc8520f2ce3f..0e85b844ef152b 100644 --- a/Tools/peg_generator/pegen/testutil.py +++ b/Tools/peg_generator/pegen/testutil.py @@ -116,7 +116,7 @@ def generate_parser_c_extension( def print_memstats() -> bool: MiB: Final = 2**20 try: - import psutil # type: ignore[import] + import psutil except ImportError: return False print("Memory stats:") diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index b814169974bd08..49f783c096f2d2 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,3 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI mypy==1.5.1 + +# needed for peg_generator: +types-psutil==5.9.5.16 +types-setuptools==68.1.0.0 From 0b0c1d046cac540deefc56ab3c38732bc76f6c56 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 29 Aug 2023 22:02:12 +0200 Subject: [PATCH 403/598] gh-108278: Deprecate passing the first param of sqlite3.Connection callback APIs by keyword (#108632) Deprecate passing the callback callable by keyword for the following sqlite3.Connection APIs: - set_authorizer(authorizer_callback) - set_progress_handler(progress_handler, ...) - set_trace_callback(trace_callback) The affected parameters will become positional-only in Python 3.15. --- Doc/library/sqlite3.rst | 27 +++-- Doc/whatsnew/3.13.rst | 7 ++ Lib/test/test_sqlite3/test_hooks.py | 24 +++++ Lib/test/test_sqlite3/test_userfunctions.py | 33 ++++++ ...-08-29-11-29-15.gh-issue-108278.-UhsnJ.rst | 10 ++ Modules/_sqlite/clinic/connection.c.h | 102 ++++++++++++++++-- Modules/_sqlite/connection.c | 25 +++-- 7 files changed, 206 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-29-11-29-15.gh-issue-108278.-UhsnJ.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5137e173e23740..0abdab52340dfd 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -763,10 +763,10 @@ Connection objects ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) - .. versionchanged:: 3.13 + .. versionchanged:: 3.13 - Passing *name*, *narg*, and *func* as keyword arguments is deprecated. - These parameters will become positional-only in Python 3.15. + Passing *name*, *narg*, and *func* as keyword arguments is deprecated. + These parameters will become positional-only in Python 3.15. .. method:: create_aggregate(name, n_arg, aggregate_class) @@ -822,10 +822,10 @@ Connection objects 3 - .. versionchanged:: 3.13 + .. versionchanged:: 3.13 - Passing *name*, *n_arg*, and *aggregate_class* as keyword arguments is deprecated. - These parameters will become positional-only in Python 3.15. + Passing *name*, *n_arg*, and *aggregate_class* as keyword arguments is deprecated. + These parameters will become positional-only in Python 3.15. .. method:: create_window_function(name, num_params, aggregate_class, /) @@ -991,6 +991,11 @@ Connection objects .. versionchanged:: 3.11 Added support for disabling the authorizer using ``None``. + .. versionchanged:: 3.13 + + Passing *authorizer_callback* as a keyword argument to is deprecated. + The parameter will become positional-only in Python 3.15. + .. method:: set_progress_handler(progress_handler, n) @@ -1006,6 +1011,11 @@ Connection objects currently executing query and cause it to raise a :exc:`DatabaseError` exception. + .. versionchanged:: 3.13 + + Passing *progress_handler* as a keyword argument to is deprecated. + The parameter will become positional-only in Python 3.15. + .. method:: set_trace_callback(trace_callback) @@ -1030,6 +1040,11 @@ Connection objects .. versionadded:: 3.3 + .. versionchanged:: 3.13 + + Passing *trace_callback* as a keyword argument to is deprecated. + The parameter will become positional-only in Python 3.15. + .. method:: enable_load_extension(enabled, /) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 1c94da23bc1b70..be5bf9a7ad5aed 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -266,6 +266,13 @@ Deprecated * :meth:`~sqlite3.Connection.create_function` * :meth:`~sqlite3.Connection.create_aggregate` + Deprecate passing the callback callable by keyword for the following + :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.set_authorizer` + * :meth:`~sqlite3.Connection.set_progress_handler` + * :meth:`~sqlite3.Connection.set_trace_callback` + The affected parameters will become positional-only in Python 3.15. (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 33f0af99532a10..49e72f8fcfbcbd 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -219,6 +219,18 @@ def bad_progress(): create table foo(a, b) """) + def test_progress_handler_keyword_args(self): + regex = ( + r"Passing keyword argument 'progress_handler' to " + r"_sqlite3.Connection.set_progress_handler\(\) is deprecated. " + r"Parameter 'progress_handler' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_progress_handler(progress_handler=lambda: None, n=1) + self.assertEqual(cm.filename, __file__) + class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): @@ -340,6 +352,18 @@ def test_trace_bad_handler(self): cx.set_trace_callback(lambda stmt: 5/0) cx.execute("select 1") + def test_trace_keyword_args(self): + regex = ( + r"Passing keyword argument 'trace_callback' to " + r"_sqlite3.Connection.set_trace_callback\(\) is deprecated. " + r"Parameter 'trace_callback' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_trace_callback(trace_callback=lambda: None) + self.assertEqual(cm.filename, __file__) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index d86b8c6025f671..09019498fd5682 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -737,6 +737,27 @@ def test_aggr_text(self): val = cur.fetchone()[0] self.assertEqual(val, txt) + def test_agg_keyword_args(self): + regex = ( + r"Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' to " + r"_sqlite3.Connection.create_aggregate\(\) is deprecated. " + r"Parameters 'name', 'n_arg' and 'aggregate_class' will become " + r"positional-only in Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate("test", 1, aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate("test", n_arg=1, aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate(name="test", n_arg=0, + aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + class AuthorizerTests(unittest.TestCase): @staticmethod @@ -779,6 +800,18 @@ def test_clear_authorizer(self): self.con.execute("select * from t2") self.con.execute("select c2 from t1") + def test_authorizer_keyword_args(self): + regex = ( + r"Passing keyword argument 'authorizer_callback' to " + r"_sqlite3.Connection.set_authorizer\(\) is deprecated. " + r"Parameter 'authorizer_callback' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_authorizer(authorizer_callback=lambda: None) + self.assertEqual(cm.filename, __file__) + class AuthorizerRaiseExceptionTests(AuthorizerTests): @staticmethod diff --git a/Misc/NEWS.d/next/Library/2023-08-29-11-29-15.gh-issue-108278.-UhsnJ.rst b/Misc/NEWS.d/next/Library/2023-08-29-11-29-15.gh-issue-108278.-UhsnJ.rst new file mode 100644 index 00000000000000..fa2dbdf88b31c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-29-11-29-15.gh-issue-108278.-UhsnJ.rst @@ -0,0 +1,10 @@ +Deprecate passing the callback callable by keyword for the following +:class:`sqlite3.Connection` APIs: + +* :meth:`~sqlite3.Connection.set_authorizer` +* :meth:`~sqlite3.Connection.set_progress_handler` +* :meth:`~sqlite3.Connection.set_trace_callback` + +The affected parameters will become positional-only in Python 3.15. + +Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 992b2e6c72ea56..f9510f47dd9efa 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -712,7 +712,12 @@ PyDoc_STRVAR(pysqlite_connection_set_authorizer__doc__, "set_authorizer($self, /, authorizer_callback)\n" "--\n" "\n" -"Sets authorizer callback."); +"Set authorizer callback.\n" +"\n" +"Note: Passing keyword argument \'authorizer_callback\' to\n" +"_sqlite3.Connection.set_authorizer() is deprecated. Parameter\n" +"\'authorizer_callback\' will become positional-only in Python 3.15.\n" +""); #define PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF \ {"set_authorizer", _PyCFunction_CAST(pysqlite_connection_set_authorizer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_authorizer__doc__}, @@ -722,13 +727,24 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.set_authorizer'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.set_authorizer'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.set_authorizer'." +# endif +#endif + static PyObject * pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -758,6 +774,16 @@ pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, if (!args) { goto exit; } + if (nargs < 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'authorizer_callback' to " + "_sqlite3.Connection.set_authorizer() is deprecated. Parameter " + "'authorizer_callback' will become positional-only in Python " + "3.15.", 1)) + { + goto exit; + } + } callable = args[0]; return_value = pysqlite_connection_set_authorizer_impl(self, cls, callable); @@ -769,7 +795,22 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, "set_progress_handler($self, /, progress_handler, n)\n" "--\n" "\n" -"Sets progress handler callback."); +"Set progress handler callback.\n" +"\n" +" progress_handler\n" +" A callable that takes no arguments.\n" +" If the callable returns non-zero, the current query is terminated,\n" +" and an exception is raised.\n" +" n\n" +" The number of SQLite virtual machine instructions that are\n" +" executed between invocations of \'progress_handler\'.\n" +"\n" +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled.\n" +"\n" +"Note: Passing keyword argument \'progress_handler\' to\n" +"_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n" +"\'progress_handler\' will become positional-only in Python 3.15.\n" +""); #define PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF \ {"set_progress_handler", _PyCFunction_CAST(pysqlite_connection_set_progress_handler), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, @@ -779,13 +820,24 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.set_progress_handler'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.set_progress_handler'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.set_progress_handler'." +# endif +#endif + static PyObject * pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -816,6 +868,16 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject if (!args) { goto exit; } + if (nargs < 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'progress_handler' to " + "_sqlite3.Connection.set_progress_handler() is deprecated. " + "Parameter 'progress_handler' will become positional-only in " + "Python 3.15.", 1)) + { + goto exit; + } + } callable = args[0]; n = PyLong_AsInt(args[1]); if (n == -1 && PyErr_Occurred()) { @@ -831,7 +893,12 @@ PyDoc_STRVAR(pysqlite_connection_set_trace_callback__doc__, "set_trace_callback($self, /, trace_callback)\n" "--\n" "\n" -"Sets a trace callback called for each SQL statement (passed as unicode)."); +"Set a trace callback called for each SQL statement (passed as unicode).\n" +"\n" +"Note: Passing keyword argument \'trace_callback\' to\n" +"_sqlite3.Connection.set_trace_callback() is deprecated. Parameter\n" +"\'trace_callback\' will become positional-only in Python 3.15.\n" +""); #define PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF \ {"set_trace_callback", _PyCFunction_CAST(pysqlite_connection_set_trace_callback), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_trace_callback__doc__}, @@ -841,13 +908,24 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.set_trace_callback'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.set_trace_callback'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.set_trace_callback'." +# endif +#endif + static PyObject * pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -877,6 +955,16 @@ pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject * if (!args) { goto exit; } + if (nargs < 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'trace_callback' to " + "_sqlite3.Connection.set_trace_callback() is deprecated. " + "Parameter 'trace_callback' will become positional-only in Python" + " 3.15.", 1)) + { + goto exit; + } + } callable = args[0]; return_value = pysqlite_connection_set_trace_callback_impl(self, cls, callable); @@ -1734,4 +1822,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=f80eb1d02cf698e4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f06b254bc5c2bcaf input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 45ec0f99237b7f..24090b0b63936a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1499,17 +1499,17 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) _sqlite3.Connection.set_authorizer as pysqlite_connection_set_authorizer cls: defining_class - / authorizer_callback as callable: object + / [from 3.15] -Sets authorizer callback. +Set authorizer callback. [clinic start generated code]*/ static PyObject * pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=75fa60114fc971c3 input=605d32ba92dd3eca]*/ +/*[clinic end generated code: output=75fa60114fc971c3 input=a52bd4937c588752]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1541,18 +1541,25 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_handler cls: defining_class - / progress_handler as callable: object + A callable that takes no arguments. + If the callable returns non-zero, the current query is terminated, + and an exception is raised. + / [from 3.15] n: int + The number of SQLite virtual machine instructions that are + executed between invocations of 'progress_handler'. -Sets progress handler callback. +Set progress handler callback. + +If 'progress_handler' is None or 'n' is 0, the progress handler is disabled. [clinic start generated code]*/ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=f7c1837984bd86db]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=b4d6e2ef8b4d32f9]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1578,17 +1585,17 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, _sqlite3.Connection.set_trace_callback as pysqlite_connection_set_trace_callback cls: defining_class - / trace_callback as callable: object + / [from 3.15] -Sets a trace callback called for each SQL statement (passed as unicode). +Set a trace callback called for each SQL statement (passed as unicode). [clinic start generated code]*/ static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=d91048c03bfcee05 input=351a94210c5f81bb]*/ +/*[clinic end generated code: output=d91048c03bfcee05 input=d705d592ec03cf28]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; From d08d49dd09917531b113486976b03b5287b574f1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Aug 2023 23:08:45 +0300 Subject: [PATCH 404/598] Revert "Use non alternate name for Kyiv (GH-108533)" (GH-108649) This reverts commit 7659128b9d7a30ddbcb063bc12e2ddb0f1f119e0. It broke tests on the Debian and macOS buildbots. --- Lib/test/test_email/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index 5768c7d0eb57eb..25fa48c5ee217b 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -148,7 +148,7 @@ def test_localtime_epoch_notz_daylight_false(self): @unittest.skipUnless(os.path.exists('/usr/share/zoneinfo') or os.path.exists('/usr/lib/zoneinfo'), "Can't find the Olson's TZ database") - @test.support.run_with_tz('Europe/Kyiv') + @test.support.run_with_tz('Europe/Kiev') def test_variable_tzname(self): t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) From 14ec0bb7c363def917f768b76f334146a3cddd84 Mon Sep 17 00:00:00 2001 From: sterliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 30 Aug 2023 04:11:31 +0300 Subject: [PATCH 405/598] Mention Ellipsis pickling in the docs (#103660) --- Doc/library/pickle.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index ba00ba29f5ba48..93387fb0b45038 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -494,7 +494,8 @@ What can be pickled and unpickled? The following types can be pickled: -* ``None``, ``True``, and ``False``; +* built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and + ``NotImplemented``); * integers, floating-point numbers, complex numbers; From 24b9bdd6eaf0b04667b6cd4c8154f7c7c10c2398 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Aug 2023 05:34:43 +0200 Subject: [PATCH 406/598] gh-106320: Remove private _Py_ForgetReference() (#108664) Move the private _Py_ForgetReference() function to the internal C API (pycore_object.h). --- Include/cpython/object.h | 5 ----- Include/internal/pycore_object.h | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 489b2eecd32991..e5987191cfe08c 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -5,11 +5,6 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); -#ifdef Py_TRACE_REFS -/* Py_TRACE_REFS is such major surgery that we call external routines. */ -PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); -#endif - #ifdef Py_REF_DEBUG /* These are useful as debugging aids when chasing down refleaks. */ PyAPI_FUNC(Py_ssize_t) _Py_GetGlobalRefTotal(void); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 32ff2794e36231..7cb822e6af9750 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -31,6 +31,11 @@ extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, extern void _PyObject_DebugTypeStats(FILE *out); +#ifdef Py_TRACE_REFS +/* Py_TRACE_REFS is such major surgery that we call external routines. */ +PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); +#endif + // Export for shared _testinternalcapi extension PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); From 210a5d7b8b2f5cdaf3740e8b9b468ed5ddf24591 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 30 Aug 2023 11:41:04 +0300 Subject: [PATCH 407/598] Revert "gh-103224: Use the realpath of the Python executable in `test_venv` (GH-103243)" (GH-108667) This reverts commit 85b0b0cd947c5218260fb2bc2014c8c8de172d33. It broke builtbots. --- Lib/test/test_venv.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 3d19b2b2e905f3..5205604c2c7185 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -208,8 +208,7 @@ def test_prompt(self): def test_upgrade_dependencies(self): builder = venv.EnvBuilder() bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' - python_exe_realpath = os.path.realpath(sys._base_executable) - python_exe = os.path.split(python_exe_realpath)[1] + python_exe = os.path.split(sys.executable)[1] with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -552,8 +551,7 @@ def test_zippath_from_non_installed_posix(self): self.addCleanup(rmtree, non_installed_dir) bindir = os.path.join(non_installed_dir, self.bindir) os.mkdir(bindir) - python_exe_realpath = os.path.realpath(sys._base_executable) - shutil.copy2(python_exe_realpath, bindir) + shutil.copy2(sys.executable, bindir) libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") @@ -597,7 +595,7 @@ def test_zippath_from_non_installed_posix(self): # libpython.so ld_library_path = sysconfig.get_config_var("LIBDIR") if not ld_library_path or sysconfig.is_python_build(): - ld_library_path = os.path.abspath(os.path.dirname(python_exe_realpath)) + ld_library_path = os.path.abspath(os.path.dirname(sys.executable)) if sys.platform == 'darwin': ld_library_path_env = "DYLD_LIBRARY_PATH" else: From 400a1cebc743515e40157ed7af86e48d654290ce Mon Sep 17 00:00:00 2001 From: Corvin Date: Wed, 30 Aug 2023 05:06:21 -0400 Subject: [PATCH 408/598] gh-108590: Fix sqlite3.iterdump for invalid Unicode in TEXT columns (#108657) Co-authored-by: Erlend E. Aasland --- Lib/sqlite3/dump.py | 27 +++++++++++++++++-- Lib/test/test_sqlite3/test_dump.py | 15 +++++++++++ ...-08-29-22-53-48.gh-issue-108590.6k0pOl.rst | 1 + 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index ead3360ce67608..481d605194c7fe 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -7,6 +7,10 @@ # future enhancements, you should normally quote any identifier that # is an English language word, even if you do not have to." + +from contextlib import contextmanager + + def _quote_name(name): return '"{0}"'.format(name.replace('"', '""')) @@ -15,6 +19,24 @@ def _quote_value(value): return "'{0}'".format(value.replace("'", "''")) +def _force_decode(bs, *args, **kwargs): + # gh-108590: Don't fail if the database contains invalid Unicode data. + try: + return bs.decode(*args, **kwargs) + except UnicodeDecodeError: + return "".join([chr(c) for c in bs]) + + +@contextmanager +def _text_factory(con, factory): + saved_factory = con.text_factory + con.text_factory = factory + try: + yield + finally: + con.text_factory = saved_factory + + def _iterdump(connection): """ Returns an iterator to the dump of the database in an SQL text format. @@ -74,8 +96,9 @@ def _iterdump(connection): ) ) query_res = cu.execute(q) - for row in query_res: - yield("{0};".format(row[0])) + with _text_factory(connection, bytes): + for row in query_res: + yield("{0};".format(_force_decode(row[0]))) # Now when the type is 'index', 'trigger', or 'view' q = """ diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 3107e1b165d950..0279ce68eeb5f1 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -133,6 +133,21 @@ def test_dump_virtual_tables(self): actual = list(self.cx.iterdump()) self.assertEqual(expected, actual) + def test_dump_unicode_invalid(self): + # gh-108590 + expected = [ + "BEGIN TRANSACTION;", + "CREATE TABLE foo (data TEXT);", + "INSERT INTO \"foo\" VALUES('a\x9f');", + "COMMIT;", + ] + self.cu.executescript(""" + CREATE TABLE foo (data TEXT); + INSERT INTO foo VALUES (CAST(X'619f' AS TEXT)); + """) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst b/Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst new file mode 100644 index 00000000000000..50b41f2a94d9be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst @@ -0,0 +1 @@ +Fixed an issue where :meth:`sqlite3.Connection.iterdump` would fail and leave an incomplete SQL dump if a table includes invalid Unicode sequences. Patch by Corvin McPherson From 6c484c39beeb66d40ef0a73cc4f1e900ea498cfa Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 30 Aug 2023 11:36:59 +0200 Subject: [PATCH 409/598] gh-108669: unittest: Fix documentation for TestResult.collectedDurations (#108670) --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 518bf6b13bad54..4c28e8fae8b088 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2017,7 +2017,7 @@ Loading and running tests .. attribute:: collectedDurations - A list containing 2-tuples of :class:`TestCase` instances and floats + A list containing 2-tuples of test case names and floats representing the elapsed time of each test which was run. .. versionadded:: 3.12 From 38ab0dba801884b0963ef0daa95e94e120a2b524 Mon Sep 17 00:00:00 2001 From: kato8966 <66937409+kato8966@users.noreply.github.com> Date: Wed, 30 Aug 2023 20:05:29 +0900 Subject: [PATCH 410/598] Fix typo in multiprocessing docs (#108666) --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2efc08f130af32..38d24a86072970 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2988,7 +2988,7 @@ Global variables Safe importing of main module Make sure that the main module can be safely imported by a new Python - interpreter without causing unintended side effects (such a starting a new + interpreter without causing unintended side effects (such as starting a new process). For example, using the *spawn* or *forkserver* start method From c7cef546319c51defa01236469b636b6978b99ab Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 30 Aug 2023 05:37:45 -0600 Subject: [PATCH 411/598] gh-101100: Fix Sphinx warnings in the Logging Cookbook (#108678) --- Doc/howto/logging-cookbook.rst | 15 +++++++++++---- Doc/tools/.nitignore | 1 - 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 6ef252d709e735..772973edadd9d8 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -761,7 +761,7 @@ printed on the console; on the server side, you should see something like: Note that there are some security issues with pickle in some scenarios. If these affect you, you can use an alternative serialization scheme by overriding -the :meth:`~handlers.SocketHandler.makePickle` method and implementing your +the :meth:`~SocketHandler.makePickle` method and implementing your alternative there, as well as adapting the above script to use your alternative serialization. @@ -835,6 +835,8 @@ To test these files, do the following in a POSIX environment: You may need to tweak the configuration files in the unlikely event that the configured ports clash with something else in your test environment. +.. currentmodule:: logging + .. _context-info: Adding contextual information to your logging output @@ -1546,7 +1548,7 @@ Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`~handlers.RotatingFileHandler`:: +logging package provides a :class:`RotatingFileHandler`:: import glob import logging @@ -1594,6 +1596,8 @@ and each time it reaches the size limit it is renamed with the suffix Obviously this example sets the log length much too small as an extreme example. You would want to set *maxBytes* to an appropriate value. +.. currentmodule:: logging + .. _format-styles: Use of alternative formatting styles @@ -1840,6 +1844,7 @@ However, it should be borne in mind that each link in the chain adds run-time overhead to all logging operations, and the technique should only be used when the use of a :class:`Filter` does not provide the desired result. +.. currentmodule:: logging.handlers .. _zeromq-handlers: @@ -1917,6 +1922,8 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: :ref:`A more advanced logging tutorial ` +.. currentmodule:: logging + An example dictionary-based configuration ----------------------------------------- @@ -3918,8 +3925,8 @@ that in other languages such as Java and C#, loggers are often static class attributes. However, this pattern doesn't make sense in Python, where the module (and not the class) is the unit of software decomposition. -Adding handlers other than :class:`NullHandler` to a logger in a library -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Adding handlers other than :class:`~logging.NullHandler` to a logger in a library +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Configuring logging by adding handlers, formatters and filters is the responsibility of the application developer, not the library developer. If you diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 953d10b19e0ccc..4c131b06b61f80 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -26,7 +26,6 @@ Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst Doc/howto/isolating-extensions.rst -Doc/howto/logging-cookbook.rst Doc/howto/logging.rst Doc/howto/urllib2.rst Doc/library/__future__.rst From e012cf771b8d4f114bd8dbdabf10d1584764541d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Aug 2023 15:28:58 +0200 Subject: [PATCH 412/598] Document Python build requirements (#108646) Co-authored-by: Erlend E. Aasland Co-authored-by: Serhiy Storchaka --- Doc/using/configure.rst | 27 ++++++++++++++++++++++----- Doc/whatsnew/3.13.rst | 2 +- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 6279b0efbf9321..a16a4afffb1afe 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -5,29 +5,42 @@ Configure Python Build Requirements ================== -Features required to build CPython: +Features and minimum versions required to build CPython: * A `C11 `_ compiler. `Optional C11 features `_ are not required. +* On Windows, Microsoft Visual Studio 2017 or later is required. + * Support for `IEEE 754 `_ floating point numbers and `floating point Not-a-Number (NaN) `_. * Support for threads. -* OpenSSL 1.1.1 or newer for the :mod:`ssl` and :mod:`hashlib` modules. +* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.9 is the recommended + minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. -* On Windows, Microsoft Visual Studio 2017 or later is required. +* SQLite 3.15.2 for the :mod:`sqlite3` extension module. + +* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. + +* Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the + :file:`configure` script. + +.. versionchanged:: 3.13: + Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. On Windows, Visual Studio 2017 or later is required. + Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. .. versionchanged:: 3.10 OpenSSL 1.1.1 is now required. + Require SQLite 3.7.15. .. versionchanged:: 3.7 Thread support and OpenSSL 1.0.2 are now required. @@ -37,7 +50,11 @@ Features required to build CPython: inline`` functions. .. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is required. + On Windows, Visual Studio 2015 or later is now required. + Tcl/Tk version 8.4 is now required. + +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". @@ -48,7 +65,7 @@ support". Configure Options ================= -List all ``./configure`` script options using:: +List all :file:`configure` script options using:: ./configure --help diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index be5bf9a7ad5aed..298d5fb567732c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -822,7 +822,7 @@ Build Changes ============= * Autoconf 2.71 and aclocal 1.16.4 is now required to regenerate - :file:`!configure`. + the :file:`configure` script. (Contributed by Christian Heimes in :gh:`89886`.) * SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. From 2928e5dc6512e4206c616cd33e0bcc3288abf6ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Aug 2023 16:02:48 +0200 Subject: [PATCH 413/598] gh-108494: Argument Clinic: Document how to generate code that uses the limited C API (#108584) Co-authored-by: Alex Waygood Co-authored-by: Erlend E. Aasland --- Doc/howto/clinic.rst | 23 ++++++++++++++++++- ...-08-25-22-40-12.gh-issue-108494.4RbDdu.rst | 3 ++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 667859e20a5d87..92701b09c3bca2 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -158,7 +158,7 @@ process a single source file, like this: The CLI supports the following options: .. program:: ./Tools/clinic/clinic.py [-h] [-f] [-o OUTPUT] [-v] \ - [--converters] [--make] [--srcdir SRCDIR] [FILE ...] + [--converters] [--make] [--srcdir SRCDIR] [--limited] [FILE ...] .. option:: -h, --help @@ -193,6 +193,11 @@ The CLI supports the following options: A file to exclude in :option:`--make` mode. This option can be given multiple times. +.. option:: --limited + + Use the :ref:`Limited API ` to parse arguments in the generated C code. + See :ref:`clinic-howto-limited-capi`. + .. option:: FILE ... The list of files to process. @@ -1905,6 +1910,22 @@ blocks embedded in Python files look slightly different. They look like this: #/*[python checksum:...]*/ +.. _clinic-howto-limited-capi: + +How to use the Limited C API +---------------------------- + +If Argument Clinic :term:`input` is located within a C source file +that contains ``#define Py_LIMITED_API``, Argument Clinic will generate C code +that uses the :ref:`Limited API ` to parse arguments. The +advantage of this is that the generated code will not use private functions. +However, this *can* result in Argument Clinic generating less efficient code +in some cases. The extent of the performance penalty will depend +on the parameters (types, number, etc.). + +.. versionadded:: 3.13 + + .. _clinic-howto-override-signature: How to override the generated signature diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst index 2d611527b26092..b96da7c0a16df8 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst @@ -1,2 +1,3 @@ :ref:`Argument Clinic ` now has a partial support of the -:ref:`Limited API `. Patch by Victor Stinner. +:ref:`Limited API `: see :ref:`clinic-howto-limited-capi`. +Patch by Victor Stinner. From 2a3926fa51b7264787d5988abf083d8c4328f4ad Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 30 Aug 2023 16:53:10 +0200 Subject: [PATCH 414/598] gh-108590: Revert gh-108657 (commit 400a1cebc) (#108686) Reverted per Serhiy's request. --- Lib/sqlite3/dump.py | 27 ++----------------- Lib/test/test_sqlite3/test_dump.py | 15 ----------- ...-08-29-22-53-48.gh-issue-108590.6k0pOl.rst | 1 - 3 files changed, 2 insertions(+), 41 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 481d605194c7fe..ead3360ce67608 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -7,10 +7,6 @@ # future enhancements, you should normally quote any identifier that # is an English language word, even if you do not have to." - -from contextlib import contextmanager - - def _quote_name(name): return '"{0}"'.format(name.replace('"', '""')) @@ -19,24 +15,6 @@ def _quote_value(value): return "'{0}'".format(value.replace("'", "''")) -def _force_decode(bs, *args, **kwargs): - # gh-108590: Don't fail if the database contains invalid Unicode data. - try: - return bs.decode(*args, **kwargs) - except UnicodeDecodeError: - return "".join([chr(c) for c in bs]) - - -@contextmanager -def _text_factory(con, factory): - saved_factory = con.text_factory - con.text_factory = factory - try: - yield - finally: - con.text_factory = saved_factory - - def _iterdump(connection): """ Returns an iterator to the dump of the database in an SQL text format. @@ -96,9 +74,8 @@ def _iterdump(connection): ) ) query_res = cu.execute(q) - with _text_factory(connection, bytes): - for row in query_res: - yield("{0};".format(_force_decode(row[0]))) + for row in query_res: + yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' q = """ diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 0279ce68eeb5f1..3107e1b165d950 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -133,21 +133,6 @@ def test_dump_virtual_tables(self): actual = list(self.cx.iterdump()) self.assertEqual(expected, actual) - def test_dump_unicode_invalid(self): - # gh-108590 - expected = [ - "BEGIN TRANSACTION;", - "CREATE TABLE foo (data TEXT);", - "INSERT INTO \"foo\" VALUES('a\x9f');", - "COMMIT;", - ] - self.cu.executescript(""" - CREATE TABLE foo (data TEXT); - INSERT INTO foo VALUES (CAST(X'619f' AS TEXT)); - """) - actual = list(self.cx.iterdump()) - self.assertEqual(expected, actual) - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst b/Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst deleted file mode 100644 index 50b41f2a94d9be..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-29-22-53-48.gh-issue-108590.6k0pOl.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue where :meth:`sqlite3.Connection.iterdump` would fail and leave an incomplete SQL dump if a table includes invalid Unicode sequences. Patch by Corvin McPherson From add8d45cbe46581b9748909fbbf60fdc8ee8f71e Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 30 Aug 2023 13:07:41 -0400 Subject: [PATCH 415/598] gh-108520: Fix bad fork detection in nested multiprocessing use case (#108568) gh-107275 introduced a regression where a SemLock would fail being passed along nested child processes, as the `is_fork_ctx` attribute would be left missing after the first deserialization. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Antoine Pitrou --- Lib/multiprocessing/synchronize.py | 8 +++--- Lib/test/_test_multiprocessing.py | 26 +++++++++++++++++++ ...-08-30-15-41-47.gh-issue-108520.u0ZGP_.rst | 3 +++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-30-15-41-47.gh-issue-108520.u0ZGP_.rst diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 2328d332123082..3ccbfe311c71f3 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -50,8 +50,8 @@ class SemLock(object): def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() - self.is_fork_ctx = ctx.get_start_method() == 'fork' - unlink_now = sys.platform == 'win32' or self.is_fork_ctx + self._is_fork_ctx = ctx.get_start_method() == 'fork' + unlink_now = sys.platform == 'win32' or self._is_fork_ctx for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( @@ -103,7 +103,7 @@ def __getstate__(self): if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: - if self.is_fork_ctx: + if self._is_fork_ctx: raise RuntimeError('A SemLock created in a fork context is being ' 'shared with a process in a spawn context. This is ' 'not supported. Please use the same context to create ' @@ -115,6 +115,8 @@ def __setstate__(self, state): self._semlock = _multiprocessing.SemLock._rebuild(*state) util.debug('recreated blocker with handle %r' % state[0]) self._make_methods() + # Ensure that deserialized SemLock can be serialized again (gh-108520). + self._is_fork_ctx = False @staticmethod def _make_name(): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 343e9dcf769e86..7a851d3df78781 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5443,6 +5443,32 @@ def test_mixed_startmethod(self): p.start() p.join() + @classmethod + def _put_one_in_queue(cls, queue): + queue.put(1) + + @classmethod + def _put_two_and_nest_once(cls, queue): + queue.put(2) + process = multiprocessing.Process(target=cls._put_one_in_queue, args=(queue,)) + process.start() + process.join() + + def test_nested_startmethod(self): + # gh-108520: Regression test to ensure that child process can send its + # arguments to another process + queue = multiprocessing.Queue() + + process = multiprocessing.Process(target=self._put_two_and_nest_once, args=(queue,)) + process.start() + process.join() + + results = [] + while not queue.empty(): + results.append(queue.get()) + + self.assertEqual(results, [2, 1]) + @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-30-15-41-47.gh-issue-108520.u0ZGP_.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-30-15-41-47.gh-issue-108520.u0ZGP_.rst new file mode 100644 index 00000000000000..44131fb11f068c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-30-15-41-47.gh-issue-108520.u0ZGP_.rst @@ -0,0 +1,3 @@ +Fix :meth:`multiprocessing.synchronize.SemLock.__setstate__` to properly initialize :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx`. This fixes a regression when passing a SemLock accross nested processes. + +Rename :attr:`multiprocessing.synchronize.SemLock.is_fork_ctx` to :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx` to avoid exposing it as public API. From f59c66e8c8a6715c585cf2cdf1f99715480b4da1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Aug 2023 21:33:04 +0200 Subject: [PATCH 416/598] gh-108297: Remove test_crashers (#108690) The test was skipped in 2011 by commit 89ba56d5fb1335ef808b87cd33cba95ea141c65d. Scripts in Lib/test/crashers/ do not crash on a reliable way. They rely on undefined behaviors, like state of the stack memory, and so may or may not crash. It is not worth it to make sure that they crash in a continious integration, they should be run manually time to time instead. --- Lib/test/test_crashers.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 Lib/test/test_crashers.py diff --git a/Lib/test/test_crashers.py b/Lib/test/test_crashers.py deleted file mode 100644 index 31b712028f8a12..00000000000000 --- a/Lib/test/test_crashers.py +++ /dev/null @@ -1,37 +0,0 @@ -# Tests that the crashers in the Lib/test/crashers directory actually -# do crash the interpreter as expected -# -# If a crasher is fixed, it should be moved elsewhere in the test suite to -# ensure it continues to work correctly. - -import unittest -import glob -import os.path -import test.support -from test.support.script_helper import assert_python_failure - -CRASHER_DIR = os.path.join(os.path.dirname(__file__), "crashers") -CRASHER_FILES = os.path.join(glob.escape(CRASHER_DIR), "*.py") - -infinite_loops = ["infinite_loop_re.py", "nasty_eq_vs_dict.py"] - -class CrasherTest(unittest.TestCase): - - @unittest.skip("these tests are too fragile") - @test.support.cpython_only - def test_crashers_crash(self): - for fname in glob.glob(CRASHER_FILES): - if os.path.basename(fname) in infinite_loops: - continue - # Some "crashers" only trigger an exception rather than a - # segfault. Consider that an acceptable outcome. - if test.support.verbose: - print("Checking crasher:", fname) - assert_python_failure(fname) - - -def tearDownModule(): - test.support.reap_children() - -if __name__ == "__main__": - unittest.main() From d52c4482a82f3f98f1a78efa948144a1fe3c52b2 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 30 Aug 2023 17:50:50 -0600 Subject: [PATCH 417/598] gh-108654: restore comprehension locals before handling exception (#108659) Co-authored-by: Dong-hee Na --- Lib/test/test_listcomps.py | 35 ++++++++++ ...-08-29-17-53-12.gh-issue-108654.jbkDVo.rst | 2 + Python/compile.c | 67 +++++++++++++++---- 3 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-29-17-53-12.gh-issue-108654.jbkDVo.rst diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 9f28ced32bd26c..bedd99b4a44fcb 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -561,6 +561,41 @@ def test_iter_var_available_in_locals(self): } ) + def test_comp_in_try_except(self): + template = """ + value = ["a"] + try: + [{func}(value) for value in value] + except: + pass + """ + for func in ["str", "int"]: + code = template.format(func=func) + raises = func != "str" + with self.subTest(raises=raises): + self._check_in_scopes(code, {"value": ["a"]}) + + def test_comp_in_try_finally(self): + code = """ + def f(value): + try: + [{func}(value) for value in value] + finally: + return value + ret = f(["a"]) + """ + self._check_in_scopes(code, {"ret": ["a"]}) + + def test_exception_in_post_comp_call(self): + code = """ + value = [1, None] + try: + [v for v in value].sort() + except: + pass + """ + self._check_in_scopes(code, {"value": [1, None]}) + __test__ = {'doctests' : doctests} diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-29-17-53-12.gh-issue-108654.jbkDVo.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-29-17-53-12.gh-issue-108654.jbkDVo.rst new file mode 100644 index 00000000000000..032e0331b20e75 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-29-17-53-12.gh-issue-108654.jbkDVo.rst @@ -0,0 +1,2 @@ +Restore locals shadowed by an inlined comprehension if the comprehension +raises an exception. diff --git a/Python/compile.c b/Python/compile.c index 6b816b4c6eda6c..50e29b4607f7ce 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5529,6 +5529,8 @@ typedef struct { PyObject *pushed_locals; PyObject *temp_symbols; PyObject *fast_hidden; + jump_target_label cleanup; + jump_target_label end; } inlined_comprehension_state; static int @@ -5639,11 +5641,45 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // `pushed_locals` on the stack, but this will be reversed when we swap // out the comprehension result in pop_inlined_comprehension_state ADDOP_I(c, loc, SWAP, PyList_GET_SIZE(state->pushed_locals) + 1); + + // Add our own cleanup handler to restore comprehension locals in case + // of exception, so they have the correct values inside an exception + // handler or finally block. + NEW_JUMP_TARGET_LABEL(c, cleanup); + state->cleanup = cleanup; + NEW_JUMP_TARGET_LABEL(c, end); + state->end = end; + + // no need to push an fblock for this "virtual" try/finally; there can't + // be return/continue/break inside a comprehension + ADDOP_JUMP(c, loc, SETUP_FINALLY, cleanup); } return SUCCESS; } +static int +restore_inlined_comprehension_locals(struct compiler *c, location loc, + inlined_comprehension_state state) +{ + PyObject *k; + // pop names we pushed to stack earlier + Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals); + // Preserve the comprehension result (or exception) as TOS. This + // reverses the SWAP we did in push_inlined_comprehension_state to get + // the outermost iterable to TOS, so we can still just iterate + // pushed_locals in simple reverse order + ADDOP_I(c, loc, SWAP, npops + 1); + for (Py_ssize_t i = npops - 1; i >= 0; --i) { + k = PyList_GetItem(state.pushed_locals, i); + if (k == NULL) { + return ERROR; + } + ADDOP_NAME(c, loc, STORE_FAST_MAYBE_NULL, k, varnames); + } + return SUCCESS; +} + static int pop_inlined_comprehension_state(struct compiler *c, location loc, inlined_comprehension_state state) @@ -5660,19 +5696,22 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, Py_CLEAR(state.temp_symbols); } if (state.pushed_locals) { - // pop names we pushed to stack earlier - Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals); - // Preserve the list/dict/set result of the comprehension as TOS. This - // reverses the SWAP we did in push_inlined_comprehension_state to get - // the outermost iterable to TOS, so we can still just iterate - // pushed_locals in simple reverse order - ADDOP_I(c, loc, SWAP, npops + 1); - for (Py_ssize_t i = npops - 1; i >= 0; --i) { - k = PyList_GetItem(state.pushed_locals, i); - if (k == NULL) { - return ERROR; - } - ADDOP_NAME(c, loc, STORE_FAST_MAYBE_NULL, k, varnames); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_JUMP(c, NO_LOCATION, JUMP, state.end); + + // cleanup from an exception inside the comprehension + USE_LABEL(c, state.cleanup); + // discard incomplete comprehension result (beneath exc on stack) + ADDOP_I(c, NO_LOCATION, SWAP, 2); + ADDOP(c, NO_LOCATION, POP_TOP); + if (restore_inlined_comprehension_locals(c, loc, state) < 0) { + return ERROR; + } + ADDOP_I(c, NO_LOCATION, RERAISE, 0); + + USE_LABEL(c, state.end); + if (restore_inlined_comprehension_locals(c, loc, state) < 0) { + return ERROR; } Py_CLEAR(state.pushed_locals); } @@ -5715,7 +5754,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, expr_ty val) { PyCodeObject *co = NULL; - inlined_comprehension_state inline_state = {NULL, NULL}; + inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL, NO_LABEL}; comprehension_ty outermost; int scope_type = c->u->u_scope_type; int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); From 157b89e55ed1ec12418a4853a0ba10eabc11ce60 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 30 Aug 2023 18:15:31 -0600 Subject: [PATCH 418/598] gh-108696: revert bypassing import cache in test_import helper (#108698) --- Lib/test/test_import/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 559198759c0d23..051711bfd1fe24 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -25,7 +25,6 @@ import _testinternalcapi import _imp -from test.support import import_helper from test.support import os_helper from test.support import ( STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, @@ -59,7 +58,7 @@ def _require_loader(module, loader, skip): if isinstance(module, str): - module = import_helper.import_fresh_module(module) + module = __import__(module) MODULE_KINDS = { BuiltinImporter: 'built-in', From 991e4e76b54b69f227242e73c2ec9d62f903da53 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 31 Aug 2023 10:28:27 +0300 Subject: [PATCH 419/598] gh-101100: Fix sphinx warnings in `threading.rst` (#108684) Co-authored-by: Hugo van Kemenade --- Doc/library/threading.rst | 16 ++++++++-------- Doc/tools/.nitignore | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 83ed48052704fb..23d8cd158abd5d 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -272,7 +272,7 @@ The instance's values will be different for separate threads. A class that represents thread-local data. For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module: :source:`Lib/_threading_local.py`. + :mod:`!_threading_local` module: :source:`Lib/_threading_local.py`. .. _thread-objects: @@ -285,7 +285,7 @@ thread of control. There are two ways to specify the activity: by passing a callable object to the constructor, or by overriding the :meth:`~Thread.run` method in a subclass. No other methods (except for the constructor) should be overridden in a subclass. In other words, *only* override the -:meth:`~Thread.__init__` and :meth:`~Thread.run` methods of this class. +``__init__()`` and :meth:`~Thread.run` methods of this class. Once a thread object is created, its activity must be started by calling the thread's :meth:`~Thread.start` method. This invokes the :meth:`~Thread.run` @@ -337,7 +337,7 @@ since it is impossible to detect the termination of alien threads. are: *group* should be ``None``; reserved for future extension when a - :class:`ThreadGroup` class is implemented. + :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. Defaults to ``None``, meaning nothing is called. @@ -1009,7 +1009,7 @@ This class represents an action that should be run only after a certain amount of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` and as such also functions as an example of creating custom threads. -Timers are started, as with threads, by calling their :meth:`~Timer.start` +Timers are started, as with threads, by calling their :meth:`Timer.start ` method. The timer can be stopped (before its action has begun) by calling the :meth:`~Timer.cancel` method. The interval the timer will wait before executing its action may not be exactly the same as the interval specified by @@ -1147,10 +1147,10 @@ As an example, here is a simple way to synchronize a client and server thread:: Using locks, conditions, and semaphores in the :keyword:`!with` statement ------------------------------------------------------------------------- -All of the objects provided by this module that have :meth:`acquire` and -:meth:`release` methods can be used as context managers for a :keyword:`with` -statement. The :meth:`acquire` method will be called when the block is -entered, and :meth:`release` will be called when the block is exited. Hence, +All of the objects provided by this module that have ``acquire`` and +``release`` methods can be used as context managers for a :keyword:`with` +statement. The ``acquire`` method will be called when the block is +entered, and ``release`` will be called when the block is exited. Hence, the following snippet:: with some_lock: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4c131b06b61f80..22df73ef0cfb06 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -124,7 +124,6 @@ Doc/library/tarfile.rst Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst -Doc/library/threading.rst Doc/library/time.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst From b89b838ebc817e5fbffad1ad8e1a85aa2d9f3113 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 31 Aug 2023 08:41:25 +0100 Subject: [PATCH 420/598] gh-108455: peg_generator: use `types-setuptools==68.1.0.1` in CI (#108697) --- Tools/peg_generator/mypy.ini | 5 ----- Tools/peg_generator/pegen/build.py | 9 ++++----- Tools/requirements-dev.txt | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 55429dd76eba68..8820d77b36f646 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -13,8 +13,3 @@ enable_error_code = truthy-bool,ignore-without-code # except for a few settings that can't yet be enabled: warn_return_any = False warn_unreachable = False - -[mypy-pegen.build] -# we need this for now due to some missing annotations -# in typeshed's stubs for setuptools -disallow_untyped_calls = False diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 998cd41428db19..6b04ae9ec7025c 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -148,8 +148,7 @@ def compile_c_extension( cmd.include_dirs = include_dirs if build_dir: cmd.build_temp = build_dir - # A deficiency in typeshed's stubs means we have to type: ignore: - cmd.ensure_finalized() # type: ignore[attr-defined] + cmd.ensure_finalized() compiler = new_compiler() customize_compiler(compiler) @@ -160,8 +159,8 @@ def compile_c_extension( library_filename = compiler.library_filename(extension_name, output_dir=library_dir) if newer_group(common_sources, library_filename, "newer"): if sys.platform == "win32": - # A deficiency in typeshed's stubs means we have to type: ignore: - pdb = compiler.static_lib_format % (extension_name, ".pdb") # type: ignore[attr-defined] + assert compiler.static_lib_format + pdb = compiler.static_lib_format % (extension_name, ".pdb") compile_opts = [f"/Fd{library_dir}\\{pdb}"] compile_opts.extend(extra_compile_args) else: @@ -213,7 +212,7 @@ def compile_c_extension( ext_path, libraries=cmd.get_libraries(extension), extra_postargs=extra_link_args, - export_symbols=cmd.get_export_symbols(extension), + export_symbols=cmd.get_export_symbols(extension), # type: ignore[no-untyped-call] debug=cmd.debug, build_temp=cmd.build_temp, ) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 49f783c096f2d2..35bceb205e8a9b 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.5.1 # needed for peg_generator: types-psutil==5.9.5.16 -types-setuptools==68.1.0.0 +types-setuptools==68.1.0.1 From 9c03215a3ee31462045b2c2ee162bdda30c54572 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 09:56:06 +0200 Subject: [PATCH 421/598] gh-107149: Make PyUnstable_ExecutableKinds public (#108440) Move PyUnstable_ExecutableKinds and associated macros from the internal C API to the public C API. Rename constants: replace "PY_" prefix with "PyUnstable_" prefix. --- Include/cpython/pyframe.h | 8 ++++++++ Include/internal/pycore_frame.h | 8 -------- Python/frame.c | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index 0e2afff925e31f..c5adbbe4868f69 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -33,3 +33,11 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame /* Returns the currently executing line number, or -1 if there is no line number. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); + +#define PyUnstable_EXECUTABLE_KIND_SKIP 0 +#define PyUnstable_EXECUTABLE_KIND_PY_FUNCTION 1 +#define PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION 3 +#define PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4 +#define PyUnstable_EXECUTABLE_KINDS 5 + +PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1]; diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index ae77367f6a3c9b..c8fad1562d8443 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -311,14 +311,6 @@ PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) return (PyGenObject *)(((char *)frame) - offset_in_gen); } -#define PY_EXECUTABLE_KIND_SKIP 0 -#define PY_EXECUTABLE_KIND_PY_FUNCTION 1 -#define PY_EXECUTABLE_KIND_BUILTIN_FUNCTION 3 -#define PY_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4 -#define PY_EXECUTABLE_KINDS 5 - -PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PY_EXECUTABLE_KINDS+1]; - #ifdef __cplusplus } #endif diff --git a/Python/frame.c b/Python/frame.c index fbfa54398c72b6..b483903fdf3018 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -167,10 +167,10 @@ PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) return PyCode_Addr2Line(_PyFrame_GetCode(frame), addr); } -const PyTypeObject *const PyUnstable_ExecutableKinds[PY_EXECUTABLE_KINDS+1] = { - [PY_EXECUTABLE_KIND_SKIP] = &_PyNone_Type, - [PY_EXECUTABLE_KIND_PY_FUNCTION] = &PyCode_Type, - [PY_EXECUTABLE_KIND_BUILTIN_FUNCTION] = &PyMethod_Type, - [PY_EXECUTABLE_KIND_METHOD_DESCRIPTOR] = &PyMethodDescr_Type, - [PY_EXECUTABLE_KINDS] = NULL, +const PyTypeObject *const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1] = { + [PyUnstable_EXECUTABLE_KIND_SKIP] = &_PyNone_Type, + [PyUnstable_EXECUTABLE_KIND_PY_FUNCTION] = &PyCode_Type, + [PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION] = &PyMethod_Type, + [PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR] = &PyMethodDescr_Type, + [PyUnstable_EXECUTABLE_KINDS] = NULL, }; From 194c6fb85e02ecbb1eedb26601e9d503c4d25174 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 11:15:31 +0200 Subject: [PATCH 422/598] gh-106320: Don't export _Py_ForgetReference() function (#108712) There is no need to export the _Py_ForgetReference() function of the Py_TRACE_REFS build. It's not used by shared extensions. Enhance also its comment. --- Include/internal/pycore_object.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7cb822e6af9750..7c142b384d17fd 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -32,8 +32,14 @@ extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, extern void _PyObject_DebugTypeStats(FILE *out); #ifdef Py_TRACE_REFS -/* Py_TRACE_REFS is such major surgery that we call external routines. */ -PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); +// Forget a reference registered by _Py_NewReference(). Function called by +// _Py_Dealloc(). +// +// On a free list, the function can be used before modifying an object to +// remove the object from traced objects. Then _Py_NewReference() or +// _Py_NewReferenceNoTotal() should be called again on the object to trace +// it again. +extern void _Py_ForgetReference(PyObject *); #endif // Export for shared _testinternalcapi extension From 059bd4d299b384d2b434ccb106f91cb4bb03bbb1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 31 Aug 2023 11:34:52 +0100 Subject: [PATCH 423/598] GH-108614: Remove non-debug uses of `#if TIER_ONE` and `#if TIER_TWO` from `_POP_FRAME` op. (GH-108685) --- Python/bytecodes.c | 23 +++++++------- Python/ceval.c | 51 +++++++++++++++++++------------ Python/ceval_macros.h | 36 ++++++++++++++++++++++ Python/executor.c | 2 +- Python/executor_cases.c.h | 23 +++++++------- Python/generated_cases.c.h | 50 +++++++++++++++++------------- Tools/cases_generator/stacking.py | 4 +-- 7 files changed, 123 insertions(+), 66 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 93926c03421eb7..7f398391c5dc34 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -768,24 +768,25 @@ dummy_func( // different frame, and it's accounted for by _PUSH_FRAME. op(_POP_FRAME, (retval --)) { assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; #if TIER_ONE assert(frame != &entry_frame); #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); - #if TIER_ONE - goto resume_frame; - #endif - #if TIER_TWO - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + LOAD_SP(); + LOAD_IP(); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif } macro(RETURN_VALUE) = diff --git a/Python/ceval.c b/Python/ceval.c index a56d31ea073639..a22852ec13ea99 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -189,6 +189,34 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) fflush(stdout); PyErr_SetRaisedException(exc); } + +static int +maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip_frame, PyObject *globals) +{ + if (globals == NULL) { + return 0; + } + if (frame == skip_frame) { + return 0; + } + int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); + if (r < 0) { + return -1; + } + int lltrace = r; + if (!lltrace) { + // When tracing executed uops, also trace bytecode + char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that + } + } + if (lltrace) { + lltrace_resume_frame(frame); + } + return lltrace; +} + #endif static void monitor_raise(PyThreadState *tstate, @@ -576,6 +604,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); } +#define TIER_ONE 1 #include "ceval_macros.h" @@ -714,24 +743,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int SET_LOCALS_FROM_FRAME(); #ifdef LLTRACE - { - if (frame != &entry_frame && GLOBALS()) { - int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); - if (r < 0) { - goto exit_unwind; - } - lltrace = r; - if (!lltrace) { - // When tracing executed uops, also trace bytecode - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that - } - } - } - if (lltrace) { - lltrace_resume_frame(frame); - } + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; } #endif @@ -752,7 +766,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif { -#define TIER_ONE 1 #include "generated_cases.c.h" /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 635b8e501e523e..4b7c4448e0ea25 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -373,3 +373,39 @@ static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } + + +/* Implementation of "macros" that modify the instruction pointer, + * stack pointer, or frame pointer. + * These need to treated differently by tier 1 and 2. */ + +#if TIER_ONE + +#define LOAD_IP() \ +do { next_instr = frame->prev_instr+1; } while (0) + +#define STORE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) + +#define LOAD_SP() \ +stack_pointer = _PyFrame_GetStackPointer(frame); + +#endif + + +#if TIER_TWO + +#define LOAD_IP() \ +do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0) + +#define STORE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) + +#define LOAD_SP() \ +stack_pointer = _PyFrame_GetStackPointer(frame); + +#endif + + + + diff --git a/Python/executor.c b/Python/executor.c index 0ff5106fe446ff..9b3262ed4165f1 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -17,6 +17,7 @@ #include "pycore_sliceobject.h" #include "pycore_uops.h" +#define TIER_TWO 2 #include "ceval_macros.h" @@ -83,7 +84,6 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject OBJECT_STAT_INC(optimization_uops_executed); switch (opcode) { -#define TIER_TWO 2 #include "executor_cases.c.h" default: diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1283cc7ebbf9c4..e63a36d61579a4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -690,24 +690,25 @@ retval = stack_pointer[-1]; STACK_SHRINK(1); assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; #if TIER_ONE assert(frame != &entry_frame); #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); - #if TIER_ONE - goto resume_frame; - #endif - #if TIER_TWO - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + LOAD_SP(); + LOAD_IP(); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5940c185817604..a7bf20b9d1c7f6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -990,25 +990,27 @@ STACK_SHRINK(1); { assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; #if TIER_ONE assert(frame != &entry_frame); #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); - #if TIER_ONE - goto resume_frame; - #endif - #if TIER_TWO - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + LOAD_SP(); + LOAD_IP(); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } + DISPATCH(); } TARGET(INSTRUMENTED_RETURN_VALUE) { @@ -1055,25 +1057,27 @@ retval = value; { assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; #if TIER_ONE assert(frame != &entry_frame); #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); - #if TIER_ONE - goto resume_frame; - #endif - #if TIER_TWO - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + LOAD_SP(); + LOAD_IP(); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } + DISPATCH(); } TARGET(INSTRUMENTED_RETURN_CONST) { @@ -3887,6 +3891,7 @@ ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; #endif } + DISPATCH(); } TARGET(CALL_PY_EXACT_ARGS) { @@ -3963,6 +3968,7 @@ ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; #endif } + DISPATCH(); } TARGET(CALL_PY_WITH_DEFAULTS) { diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 2d006ba8e5934a..dcdfc7ec054248 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -369,8 +369,8 @@ def write_macro_instr( next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err - if not parts[-1].instr.always_exits and not next_instr_is_set: - if mac.cache_offset: + if not parts[-1].instr.always_exits: + if not next_instr_is_set and mac.cache_offset: out.emit(f"next_instr += {mac.cache_offset};") out.emit("DISPATCH();") From 79823c103b66030f10e07e04a5462f101674a4fc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 13:53:19 +0200 Subject: [PATCH 424/598] gh-106320: Remove private _PyErr_ChainExceptions() (#108713) Remove _PyErr_ChainExceptions(), _PyErr_ChainExceptions1() and _PyErr_SetFromPyStatus() functions from the public C API. * Move the private _PyErr_ChainExceptions() and _PyErr_ChainExceptions1() function to the internal C API (pycore_pyerrors.h). * Move the private _PyErr_SetFromPyStatus() to the internal C API (pycore_initconfig.h). * No longer export the _PyErr_ChainExceptions() function. * Move run_in_subinterp_with_config() from _testcapi to _testinternalcapi. --- Include/cpython/initconfig.h | 1 - Include/cpython/pyerrors.h | 5 -- Include/internal/pycore_initconfig.h | 4 ++ Include/internal/pycore_pyerrors.h | 7 ++ Lib/test/support/__init__.py | 4 +- Lib/test/test_import/__init__.py | 8 +-- Modules/_io/_iomodule.c | 1 + Modules/_io/fileio.c | 1 + Modules/_io/iobase.c | 2 + Modules/_io/textio.c | 3 +- Modules/_sqlite/connection.c | 2 + Modules/_ssl.c | 1 + Modules/_testcapimodule.c | 101 -------------------------- Modules/_testinternalcapi.c | 103 +++++++++++++++++++++++++++ Modules/_xxsubinterpretersmodule.c | 6 ++ Modules/_zoneinfo.c | 2 +- Objects/odictobject.c | 5 +- Objects/weakrefobject.c | 1 + 18 files changed, 140 insertions(+), 117 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index cbae97f12f5377..7fb7a9868be926 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -25,7 +25,6 @@ PyAPI_FUNC(PyStatus) PyStatus_Exit(int exitcode); PyAPI_FUNC(int) PyStatus_IsError(PyStatus err); PyAPI_FUNC(int) PyStatus_IsExit(PyStatus err); PyAPI_FUNC(int) PyStatus_Exception(PyStatus err); -PyAPI_FUNC(PyObject *) _PyErr_SetFromPyStatus(PyStatus status); /* --- PyWideStringList ------------------------------------------------ */ diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index cbe3be158b38be..9633a5407f28a6 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,11 +88,6 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif -/* Context manipulation (PEP 3134) */ - -Py_DEPRECATED(3.12) PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); - /* In exceptions.c */ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c9645c7dc66573..6439101ff390ae 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -44,6 +44,10 @@ struct pyruntimestate; #define _PyStatus_UPDATE_FUNC(err) \ do { (err).func = _PyStatus_GET_FUNC(); } while (0) +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject *) _PyErr_SetFromPyStatus(PyStatus status); + + /* --- PyWideStringList ------------------------------------------------ */ #define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL} diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 0bc20589721f35..0f16fb894d17e1 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -163,6 +163,13 @@ PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, void _PyErr_FormatNote(const char *format, ...); +/* Context manipulation (PEP 3134) */ + +Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); + +// Export for '_zoneinfo' shared extension +PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); + #ifdef __cplusplus } #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 328bddbdc6887b..c3f8527bd69525 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1821,11 +1821,11 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): module is enabled. """ _check_tracemalloc() - import _testcapi + import _testinternalcapi if own_gil is not None: assert 'gil' not in config, (own_gil, config) config['gil'] = 2 if own_gil else 1 - return _testcapi.run_in_subinterp_with_config(code, **config) + return _testinternalcapi.run_in_subinterp_with_config(code, **config) def _check_tracemalloc(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 051711bfd1fe24..740ce7d5ef2638 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1805,12 +1805,12 @@ def check_compatible_fresh(self, name, *, strict=False, isolated=False): check_multi_interp_extensions=strict, ) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' - import _testcapi, sys + import _testinternalcapi, sys assert ( {name!r} in sys.builtin_module_names or {name!r} not in sys.modules ), repr({name!r}) - ret = _testcapi.run_in_subinterp_with_config( + ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, **{kwargs}, ) @@ -1829,9 +1829,9 @@ def check_incompatible_fresh(self, name, *, isolated=False): check_multi_interp_extensions=True, ) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' - import _testcapi, sys + import _testinternalcapi, sys assert {name!r} not in sys.modules, {name!r} - ret = _testcapi.run_in_subinterp_with_config( + ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, **{kwargs}, ) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 0762e26a36c795..cc844527da5838 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "_iomodule.h" diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 15df1b2befda82..fb416700e22523 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include #ifdef HAVE_SYS_TYPES_H diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 55508a2214f84d..34fcd702391f32 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -12,6 +12,8 @@ #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyType_HasFeature() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() + #include // offsetof() #include "_iomodule.h" diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 8e44d453269a71..0a727a6e0ecd8a 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -9,10 +9,11 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_codecs.h" // _PyCodecInfo_GetIncrementalDecoder() +#include "pycore_fileutils.h" // _Py_GetLocaleEncoding() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_fileutils.h" // _Py_GetLocaleEncoding() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "_iomodule.h" diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 24090b0b63936a..21bdbc12814698 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -33,7 +33,9 @@ #include "blob.h" #include "prepare_protocol.h" #include "util.h" + #include "pycore_import.h" // _PyImport_GetModuleAttrString() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_weakref.h" // _PyWeakref_IS_DEAD() diff --git a/Modules/_ssl.c b/Modules/_ssl.c index be033256adc259..cecc3785c7661c 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -27,6 +27,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _PyIsSelectable_fd() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_weakref.h" // _PyWeakref_GET_REF() /* Include symbols from _socket module */ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 20b96320f4c339..d5c8a9d7ae520d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1434,104 +1434,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -/* To run some code in a sub-interpreter. */ -static PyObject * -run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) -{ - const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; - - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { - return NULL; - } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); - return NULL; - } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); - return NULL; - } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); - return NULL; - } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); - return NULL; - } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); - return NULL; - } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); - return NULL; - } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); - return NULL; - } - - mainstate = PyThreadState_Get(); - - PyThreadState_Swap(NULL); - - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; - PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - PyThreadState_Swap(mainstate); - _PyErr_SetFromPyStatus(status); - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); - _PyErr_ChainExceptions1(exc); - return NULL; - } - assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); - Py_EndInterpreter(substate); - - PyThreadState_Swap(mainstate); - - return PyLong_FromLong(r); -} - static void _xid_capsule_destructor(PyObject *capsule) { @@ -3376,9 +3278,6 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"run_in_subinterp_with_config", - _PyCFunction_CAST(run_in_subinterp_with_config), - METH_VARARGS | METH_KEYWORDS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3e3dfeca037c7f..e375ca8556d217 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -25,6 +25,7 @@ #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -1593,6 +1594,105 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +/* To run some code in a sub-interpreter. */ +static PyObject * +run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *code; + int use_main_obmalloc = -1; + int allow_fork = -1; + int allow_exec = -1; + int allow_threads = -1; + int allow_daemon_threads = -1; + int check_multi_interp_extensions = -1; + int gil = -1; + int r; + PyThreadState *substate, *mainstate; + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + + static char *kwlist[] = {"code", + "use_main_obmalloc", + "allow_fork", + "allow_exec", + "allow_threads", + "allow_daemon_threads", + "check_multi_interp_extensions", + "gil", + NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "s$ppppppi:run_in_subinterp_with_config", kwlist, + &code, &use_main_obmalloc, + &allow_fork, &allow_exec, + &allow_threads, &allow_daemon_threads, + &check_multi_interp_extensions, + &gil)) { + return NULL; + } + if (use_main_obmalloc < 0) { + PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); + return NULL; + } + if (allow_fork < 0) { + PyErr_SetString(PyExc_ValueError, "missing allow_fork"); + return NULL; + } + if (allow_exec < 0) { + PyErr_SetString(PyExc_ValueError, "missing allow_exec"); + return NULL; + } + if (allow_threads < 0) { + PyErr_SetString(PyExc_ValueError, "missing allow_threads"); + return NULL; + } + if (gil < 0) { + PyErr_SetString(PyExc_ValueError, "missing gil"); + return NULL; + } + if (allow_daemon_threads < 0) { + PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + return NULL; + } + if (check_multi_interp_extensions < 0) { + PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + return NULL; + } + + mainstate = PyThreadState_Get(); + + PyThreadState_Swap(NULL); + + const PyInterpreterConfig config = { + .use_main_obmalloc = use_main_obmalloc, + .allow_fork = allow_fork, + .allow_exec = allow_exec, + .allow_threads = allow_threads, + .allow_daemon_threads = allow_daemon_threads, + .check_multi_interp_extensions = check_multi_interp_extensions, + .gil = gil, + }; + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyThreadState_Swap(mainstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + assert(substate != NULL); + r = PyRun_SimpleStringFlags(code, &cflags); + Py_EndInterpreter(substate); + + PyThreadState_Swap(mainstate); + + return PyLong_FromLong(r); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1659,6 +1759,9 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"run_in_subinterp_with_config", + _PyCFunction_CAST(run_in_subinterp_with_config), + METH_VARARGS | METH_KEYWORDS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index ea91e70cad991d..6638c2c8e0636b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1,7 +1,13 @@ /* interpreters module */ /* low-level access to interpreter primitives */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "interpreteridobject.h" diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index fb0b4b40b2ad5d..09f5fd4b2ef0a9 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -4,7 +4,7 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() - +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include #include diff --git a/Objects/odictobject.c b/Objects/odictobject.c index d7a0f914d41aff..b99896319e0136 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -465,10 +465,11 @@ Potential Optimizations */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_dict.h" // _Py_dict_lookup() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include // offsetof() #include "clinic/odictobject.c.h" diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 1814c6eb69c29b..df74be6aba7244 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_weakref.h" // _PyWeakref_GET_REF() From bd58389cddb0d9f294ba4468ae990014660e23d4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 15:37:14 +0200 Subject: [PATCH 425/598] Run make regen-global-objects (#108714) --- .../pycore_global_objects_fini_generated.h | 5 ----- Include/internal/pycore_global_strings.h | 5 ----- Include/internal/pycore_runtime_init_generated.h | 5 ----- Include/internal/pycore_unicodeobject_generated.h | 15 --------------- 4 files changed, 30 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d2b6058362ca23..6361f5d1100231 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -787,10 +787,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -914,8 +912,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); @@ -1231,7 +1227,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 169528d6eb1fee..504008d67fe9cd 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -276,10 +276,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) - STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) - STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -403,8 +401,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) - STRUCT_FOR_ID(exc_type) - STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) @@ -720,7 +716,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) - STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index ccfedf9b32d2a0..8cc3287ce35e5b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -785,10 +785,8 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ - INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ - INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -912,8 +910,6 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ - INIT_ID(exc_type), \ - INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ @@ -1229,7 +1225,6 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ - INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index ff7aeba1de96bf..50400db2919a73 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -669,18 +669,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(alias); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(args); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1050,12 +1044,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(eventmask); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(exc_type); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(exc_value); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(excepthook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -2001,9 +1989,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(trace_callback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(traceback); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(trailers); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); From dd32611f4f1cff4a79135f322972dfbfaebf5d6e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 16:13:53 +0200 Subject: [PATCH 426/598] gh-106320: winconsoleio.c includes pycore_pyerrors.h (#108720) Fix compiler warning: warning C4013: '_PyErr_ChainExceptions1' undefined --- Modules/_io/winconsoleio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index e10a22cdeb678d..50b8818aad410b 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #ifdef HAVE_WINDOWS_CONSOLE_IO From 013a99a47b3299f48cf7f95aa451a116441b029c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 31 Aug 2023 15:35:23 +0100 Subject: [PATCH 427/598] gh-97850: Note in py312 whatsnew that `importlib.util.set_loader` and `importlib.util.module_for_loader` have been removed (#108719) Note in py312 whatsnew that `importlib.util.set_loader` and `importlib.util.module_for_loader` have been removed --- Doc/whatsnew/3.12.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c5a5f7e6bd0235..fb31a07930da25 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1312,8 +1312,9 @@ Removed * References to, and support for :meth:`!module_repr()` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) - * ``importlib.util.set_package`` has been removed. (Contributed by Brett - Cannon in :gh:`65961`.) + * ``importlib.util.set_package``, ``importlib.util.set_loader`` and + ``importlib.util.module_for_loader`` have all been removed. (Contributed by + Brett Cannon and Nikita Sobolev in :gh:`65961` and :gh:`97850`.) * Support for ``find_loader()`` and ``find_module()`` APIs have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) From 13a00078b81776b23b0b6add69b848382240d1f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 18:33:34 +0200 Subject: [PATCH 428/598] gh-108634: Py_TRACE_REFS uses a hash table (#108663) Python built with "configure --with-trace-refs" (tracing references) is now ABI compatible with Python release build and debug build. Moreover, it now also supports the Limited API. Change Py_TRACE_REFS build: * Remove _PyObject_EXTRA_INIT macro. * The PyObject structure no longer has two extra members (_ob_prev and _ob_next). * Use a hash table (_Py_hashtable_t) to trace references (all objects): PyInterpreterState.object_state.refchain. * Py_TRACE_REFS build is now ABI compatible with release build and debug build. * Limited C API extensions can now be built with Py_TRACE_REFS: xxlimited, xxlimited_35, _testclinic_limited. * No longer rename PyModule_Create2() and PyModule_FromDefAndSpec2() functions to PyModule_Create2TraceRefs() and PyModule_FromDefAndSpec2TraceRefs(). * _Py_PrintReferenceAddresses() is now called before finalize_interp_delete() which deletes the refchain hash table. * test_tracemalloc find_trace() now also filters by size to ignore the memory allocated by _PyRefchain_Trace(). Test changes for Py_TRACE_REFS: * Add test.support.Py_TRACE_REFS constant. * Add test_sys.test_getobjects() to test sys.getobjects() function. * test_exceptions skips test_recursion_normalizing_with_no_memory() and test_memory_error_in_PyErr_PrintEx() if Python is built with Py_TRACE_REFS. * test_repl skips test_no_memory(). * test_capi skisp test_set_nomemory(). --- Doc/c-api/typeobj.rst | 22 -- Doc/using/configure.rst | 13 +- Doc/whatsnew/3.13.rst | 9 + Include/internal/pycore_object.h | 5 +- Include/internal/pycore_object_state.h | 11 +- Include/internal/pycore_runtime_init.h | 2 +- Include/modsupport.h | 8 - Include/object.h | 21 -- Include/pyport.h | 6 - Lib/test/support/__init__.py | 5 +- Lib/test/test_capi/test_mem.py | 3 + Lib/test/test_exceptions.py | 6 + Lib/test/test_import/__init__.py | 4 +- Lib/test/test_repl.py | 4 + Lib/test/test_sys.py | 21 ++ Lib/test/test_tracemalloc.py | 20 +- ...-08-30-02-52-52.gh-issue-108634.3dpBvf.rst | 3 + ...-08-30-02-54-06.gh-issue-108634.oV3Xzk.rst | 3 + Misc/SpecialBuilds.txt | 10 +- Modules/_testcapi/parts.h | 21 -- Objects/object.c | 274 +++++++++++------- Objects/setobject.c | 1 - Objects/sliceobject.c | 1 - Objects/structseq.c | 5 +- Objects/typeobject.c | 2 +- Python/bltinmodule.c | 2 +- Python/hashtable.c | 1 - Python/pylifecycle.c | 25 +- Python/pystate.c | 4 + configure | 11 +- configure.ac | 12 +- 31 files changed, 292 insertions(+), 243 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-08-30-02-52-52.gh-issue-108634.3dpBvf.rst create mode 100644 Misc/NEWS.d/next/C API/2023-08-30-02-54-06.gh-issue-108634.oV3Xzk.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index f417c68fd1efd4..1fa3f2a6f53735 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -528,28 +528,6 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is inherited by subtypes. -.. c:member:: PyObject* PyObject._ob_next - PyObject* PyObject._ob_prev - - These fields are only present when the macro ``Py_TRACE_REFS`` is defined - (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). - - Their initialization to ``NULL`` is taken care of by the - ``PyObject_HEAD_INIT`` macro. For :ref:`statically allocated objects - `, these fields always remain ``NULL``. For :ref:`dynamically - allocated objects `, these two fields are used to link the - object into a doubly linked list of *all* live objects on the heap. - - This could be used for various debugging purposes; currently the only uses - are the :func:`sys.getobjects` function and to print the objects that are - still alive at the end of a run when the environment variable - :envvar:`PYTHONDUMPREFS` is set. - - **Inheritance:** - - These fields are not inherited by subtypes. - - PyVarObject Slots ----------------- diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index a16a4afffb1afe..fe35372603fdd8 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -425,8 +425,7 @@ See also the :ref:`Python Development Mode ` and the .. versionchanged:: 3.8 Release builds and debug builds are now ABI compatible: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option), which introduces the only ABI - incompatibility. + :option:`--with-trace-refs` option). Debug options @@ -447,8 +446,14 @@ Debug options * Add :func:`sys.getobjects` function. * Add :envvar:`PYTHONDUMPREFS` environment variable. - This build is not ABI compatible with release build (default build) or debug - build (``Py_DEBUG`` and ``Py_REF_DEBUG`` macros). + The :envvar:`PYTHONDUMPREFS` environment variable can be used to dump + objects and reference counts still alive at Python exit. + + :ref:`Statically allocated objects ` are not traced. + + .. versionchanged:: 3.13 + This build is now ABI compatible with release build and :ref:`debug build + `. .. versionadded:: 3.8 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 298d5fb567732c..fd2f2c3fff8282 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -828,6 +828,11 @@ Build Changes * SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. (Contributed by Erlend Aasland in :gh:`105875`.) +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) is now ABI compatible with Python release build and + :ref:`debug build `. + (Contributed by Victor Stinner in :gh:`108634`.) + C API Changes ============= @@ -900,6 +905,10 @@ New Features (with an underscore prefix). (Contributed by Victor Stinner in :gh:`108014`.) +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) now supports the :ref:`Limited API `. + (Contributed by Victor Stinner in :gh:`108634`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7c142b384d17fd..d842816e673553 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -55,7 +55,6 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); backwards compatible solution */ #define _PyObject_HEAD_INIT(type) \ { \ - _PyObject_EXTRA_INIT \ .ob_refcnt = _Py_IMMORTAL_REFCNT, \ .ob_type = (type) \ }, @@ -184,6 +183,8 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); extern void _PyObject_InitState(PyInterpreterState *interp); +extern void _PyObject_FiniState(PyInterpreterState *interp); +extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and @@ -302,7 +303,7 @@ extern void _PyDebug_PrintTotalRefs(void); #endif #ifdef Py_TRACE_REFS -extern void _Py_AddToAllObjects(PyObject *op, int force); +extern void _Py_AddToAllObjects(PyObject *op); extern void _Py_PrintReferences(PyInterpreterState *, FILE *); extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 65feb5af969f8b..9eac27b1a9a4e3 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_hashtable.h" // _Py_hashtable_t + struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; @@ -20,11 +22,10 @@ struct _py_object_state { Py_ssize_t reftotal; #endif #ifdef Py_TRACE_REFS - /* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ - PyObject refchain; + // Hash table storing all objects. The key is the object pointer + // (PyObject*) and the value is always the number 1 (as uintptr_t). + // See _PyRefchain_IsTraced() and _PyRefchain_Trace() functions. + _Py_hashtable_t *refchain; #endif int _not_used; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index c775a8a7e7eefa..2deba02a89f33c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -192,7 +192,7 @@ extern PyTypeObject _PyExc_MemoryError; #ifdef Py_TRACE_REFS # define _py_object_state_INIT(INTERP) \ { \ - .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ + .refchain = NULL, \ } #else # define _py_object_state_INIT(INTERP) \ diff --git a/Include/modsupport.h b/Include/modsupport.h index 88577e027b5275..7c15ab50c320e2 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -111,14 +111,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" -#ifdef Py_TRACE_REFS - /* When we are tracing reference counts, rename module creation functions so - modules compiled with incompatible settings will generate a - link-time error. */ - #define PyModule_Create2 PyModule_Create2TraceRefs - #define PyModule_FromDefAndSpec2 PyModule_FromDefAndSpec2TraceRefs -#endif - PyAPI_FUNC(PyObject *) PyModule_Create2(PyModuleDef*, int apiver); #ifdef Py_LIMITED_API diff --git a/Include/object.h b/Include/object.h index d82eb6138743b9..de2a1ce0f3c4dd 100644 --- a/Include/object.h +++ b/Include/object.h @@ -58,23 +58,6 @@ whose size is determined when the object is allocated. # define Py_REF_DEBUG #endif -#if defined(Py_LIMITED_API) && defined(Py_TRACE_REFS) -# error Py_LIMITED_API is incompatible with Py_TRACE_REFS -#endif - -#ifdef Py_TRACE_REFS -/* Define pointers to support a doubly-linked list of all live heap objects. */ -#define _PyObject_HEAD_EXTRA \ - PyObject *_ob_next; \ - PyObject *_ob_prev; - -#define _PyObject_EXTRA_INIT _Py_NULL, _Py_NULL, - -#else -# define _PyObject_HEAD_EXTRA -# define _PyObject_EXTRA_INIT -#endif - /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -130,14 +113,12 @@ check by comparing the reference count field to the immortality reference count. #ifdef Py_BUILD_CORE #define PyObject_HEAD_INIT(type) \ { \ - _PyObject_EXTRA_INIT \ { _Py_IMMORTAL_REFCNT }, \ (type) \ }, #else #define PyObject_HEAD_INIT(type) \ { \ - _PyObject_EXTRA_INIT \ { 1 }, \ (type) \ }, @@ -164,8 +145,6 @@ check by comparing the reference count field to the immortality reference count. * in addition, be cast to PyVarObject*. */ struct _object { - _PyObject_HEAD_EXTRA - #if (defined(__GNUC__) || defined(__clang__)) \ && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) // On C99 and older, anonymous union is a GCC and clang extension diff --git a/Include/pyport.h b/Include/pyport.h index 2dc241389243d3..115b54fd969287 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -684,12 +684,6 @@ extern char * _getpty(int *, int, mode_t, int); # endif #endif -/* Check that ALT_SOABI is consistent with Py_TRACE_REFS: - ./configure --with-trace-refs should must be used to define Py_TRACE_REFS */ -#if defined(ALT_SOABI) && defined(Py_TRACE_REFS) -# error "Py_TRACE_REFS ABI is not compatible with release and debug ABI" -#endif - #if defined(__ANDROID__) || defined(__VXWORKS__) // Use UTF-8 as the locale encoding, ignore the LC_CTYPE locale. // See _Py_GetLocaleEncoding(), PyUnicode_DecodeLocale() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c3f8527bd69525..16a5056a33aa12 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -779,9 +779,6 @@ def python_is_optimized(): _header = 'nP' _align = '0n' -if hasattr(sys, "getobjects"): - _header = '2P' + _header - _align = '0P' _vheader = _header + 'n' def calcobjsize(fmt): @@ -2469,3 +2466,5 @@ def adjust_int_max_str_digits(max_digits): #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', 'skipped on s390x') + +Py_TRACE_REFS = hasattr(sys, 'getobjects') diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 527000875b7241..72f23b1a34080e 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -112,6 +112,9 @@ def test_pyobject_forbidden_bytes_is_freed(self): def test_pyobject_freed_is_freed(self): self.check_pyobject_is_freed('check_pyobject_freed_is_freed') + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_set_nomemory(self): code = """if 1: import _testcapi diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 764122ed4ef783..c766f4d43311eb 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1484,6 +1484,9 @@ def recurse_in_body_and_except(): @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1652,6 +1655,9 @@ def test_unhandled(self): self.assertTrue(report.endswith("\n")) @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 740ce7d5ef2638..33bce779f6cc01 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -28,7 +28,7 @@ from test.support import os_helper from test.support import ( STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, - is_wasi, run_in_subinterp, run_in_subinterp_with_config) + is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) from test.support.os_helper import ( @@ -2555,7 +2555,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): def test_basic_multiple_interpreters_deleted_no_reset(self): # without resetting; already loaded in a deleted interpreter - if hasattr(sys, 'getobjects'): + if Py_TRACE_REFS: # It's a Py_TRACE_REFS build. # This test breaks interpreter isolation a little, # which causes problems on Py_TRACE_REF builds. diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index ddb4aa68048df1..58392f2384a3d9 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -5,6 +5,7 @@ import unittest import subprocess from textwrap import dedent +from test import support from test.support import cpython_only, has_subprocess_support, SuppressCrashReport from test.support.script_helper import kill_python @@ -59,6 +60,9 @@ def run_on_interactive_mode(source): class TestInteractiveInterpreter(unittest.TestCase): @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_no_memory(self): # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index f3608ce142fb9b..d8b684c8a008f0 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1174,6 +1174,27 @@ def test_stdlib_dir(self): self.assertEqual(os.path.normpath(sys._stdlib_dir), os.path.normpath(expected)) + @unittest.skipUnless(hasattr(sys, 'getobjects'), 'need sys.getobjects()') + def test_getobjects(self): + # sys.getobjects(0) + all_objects = sys.getobjects(0) + self.assertIsInstance(all_objects, list) + self.assertGreater(len(all_objects), 0) + + # sys.getobjects(0, MyType) + class MyType: + pass + size = 100 + my_objects = [MyType() for _ in range(size)] + get_objects = sys.getobjects(0, MyType) + self.assertEqual(len(get_objects), size) + for obj in get_objects: + self.assertIsInstance(obj, MyType) + + # sys.getobjects(3, MyType) + get_objects = sys.getobjects(3, MyType) + self.assertEqual(len(get_objects), 3) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 4af4ca3b977236..bea124521032d1 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -173,9 +173,11 @@ def test_set_traceback_limit(self): self.assertEqual(len(traceback), 1) self.assertEqual(traceback, obj_traceback) - def find_trace(self, traces, traceback): + def find_trace(self, traces, traceback, size): + # filter also by size to ignore the memory allocated by + # _PyRefchain_Trace() if Python is built with Py_TRACE_REFS. for trace in traces: - if trace[2] == traceback._frames: + if trace[2] == traceback._frames and trace[1] == size: return trace self.fail("trace not found") @@ -186,11 +188,10 @@ def test_get_traces(self): obj, obj_traceback = allocate_bytes(obj_size) traces = tracemalloc._get_traces() - trace = self.find_trace(traces, obj_traceback) + trace = self.find_trace(traces, obj_traceback, obj_size) self.assertIsInstance(trace, tuple) domain, size, traceback, length = trace - self.assertEqual(size, obj_size) self.assertEqual(traceback, obj_traceback._frames) tracemalloc.stop() @@ -208,17 +209,18 @@ def allocate_bytes4(size): # Ensure that two identical tracebacks are not duplicated tracemalloc.stop() tracemalloc.start(4) - obj_size = 123 - obj1, obj1_traceback = allocate_bytes4(obj_size) - obj2, obj2_traceback = allocate_bytes4(obj_size) + obj1_size = 123 + obj2_size = 125 + obj1, obj1_traceback = allocate_bytes4(obj1_size) + obj2, obj2_traceback = allocate_bytes4(obj2_size) traces = tracemalloc._get_traces() obj1_traceback._frames = tuple(reversed(obj1_traceback._frames)) obj2_traceback._frames = tuple(reversed(obj2_traceback._frames)) - trace1 = self.find_trace(traces, obj1_traceback) - trace2 = self.find_trace(traces, obj2_traceback) + trace1 = self.find_trace(traces, obj1_traceback, obj1_size) + trace2 = self.find_trace(traces, obj2_traceback, obj2_size) domain1, size1, traceback1, length1 = trace1 domain2, size2, traceback2, length2 = trace2 self.assertIs(traceback2, traceback1) diff --git a/Misc/NEWS.d/next/Build/2023-08-30-02-52-52.gh-issue-108634.3dpBvf.rst b/Misc/NEWS.d/next/Build/2023-08-30-02-52-52.gh-issue-108634.3dpBvf.rst new file mode 100644 index 00000000000000..d1530787067d42 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-30-02-52-52.gh-issue-108634.3dpBvf.rst @@ -0,0 +1,3 @@ +Python built with :file:`configure` :option:`--with-trace-refs` (tracing +references) is now ABI compatible with Python release build and :ref:`debug +build `. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2023-08-30-02-54-06.gh-issue-108634.oV3Xzk.rst b/Misc/NEWS.d/next/C API/2023-08-30-02-54-06.gh-issue-108634.oV3Xzk.rst new file mode 100644 index 00000000000000..0427644ad37246 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-30-02-54-06.gh-issue-108634.oV3Xzk.rst @@ -0,0 +1,3 @@ +Python built with :file:`configure` :option:`--with-trace-refs` (tracing +references) now supports the :ref:`Limited API `. Patch by +Victor Stinner. diff --git a/Misc/SpecialBuilds.txt b/Misc/SpecialBuilds.txt index 5609928284d448..78201bfbd6741d 100644 --- a/Misc/SpecialBuilds.txt +++ b/Misc/SpecialBuilds.txt @@ -43,13 +43,9 @@ Py_TRACE_REFS Build option: ``./configure --with-trace-refs``. -Turn on heavy reference debugging. This is major surgery. Every PyObject grows -two more pointers, to maintain a doubly-linked list of all live heap-allocated -objects. Most built-in type objects are not in this list, as they're statically -allocated. - -Note that because the fundamental PyObject layout changes, Python modules -compiled with Py_TRACE_REFS are incompatible with modules compiled without it. +Turn on heavy reference debugging. This is major surgery. All live +heap-allocated objects are traced in a hash table. Most built-in type objects +are not in this list, as they're statically allocated. Special gimmicks: diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 65ebf80bcd1e95..9c6d6157141801 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -1,27 +1,9 @@ #ifndef Py_TESTCAPI_PARTS_H #define Py_TESTCAPI_PARTS_H -#include "pyconfig.h" // for Py_TRACE_REFS - -// Figure out if Limited API is available for this build. If it isn't we won't -// build tests for it. -// Currently, only Py_TRACE_REFS disables Limited API. -#ifdef Py_TRACE_REFS -#undef LIMITED_API_AVAILABLE -#else -#define LIMITED_API_AVAILABLE 1 -#endif - // Always enable assertions #undef NDEBUG -#if !defined(LIMITED_API_AVAILABLE) && defined(Py_LIMITED_API) -// Limited API being unavailable means that with Py_LIMITED_API defined -// we can't even include Python.h. -// Do nothing; the .c file that defined Py_LIMITED_API should also do nothing. - -#else - #include "Python.h" int _PyTestCapi_Init_Vectorcall(PyObject *module); @@ -44,10 +26,7 @@ int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *mod); -#ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); -#endif // LIMITED_API_AVAILABLE -#endif #endif // Py_TESTCAPI_PARTS_H diff --git a/Objects/object.c b/Objects/object.c index 0d88421bf0fdc9..a4d7111a686bda 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -9,6 +9,7 @@ #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition @@ -162,44 +163,51 @@ _PyDebug_PrintTotalRefs(void) { #ifdef Py_TRACE_REFS -#define REFCHAIN(interp) &interp->object_state.refchain +#define REFCHAIN(interp) interp->object_state.refchain +#define REFCHAIN_VALUE ((void*)(uintptr_t)1) -static inline void -init_refchain(PyInterpreterState *interp) -{ - PyObject *refchain = REFCHAIN(interp); - refchain->_ob_prev = refchain; - refchain->_ob_next = refchain; -} - -/* Insert op at the front of the list of all objects. If force is true, - * op is added even if _ob_prev and _ob_next are non-NULL already. If - * force is false amd _ob_prev or _ob_next are non-NULL, do nothing. - * force should be true if and only if op points to freshly allocated, - * uninitialized memory, or you've unlinked op from the list and are - * relinking it into the front. - * Note that objects are normally added to the list via _Py_NewReference, - * which is called by PyObject_Init. Not all objects are initialized that - * way, though; exceptions include statically allocated type objects, and - * statically allocated singletons (like Py_True and Py_None). - */ -void -_Py_AddToAllObjects(PyObject *op, int force) +bool +_PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj) { -#ifdef Py_DEBUG - if (!force) { - /* If it's initialized memory, op must be in or out of - * the list unambiguously. - */ - _PyObject_ASSERT(op, (op->_ob_prev == NULL) == (op->_ob_next == NULL)); + return (_Py_hashtable_get(REFCHAIN(interp), obj) == REFCHAIN_VALUE); +} + + +static void +_PyRefchain_Trace(PyInterpreterState *interp, PyObject *obj) +{ + if (_Py_hashtable_set(REFCHAIN(interp), obj, REFCHAIN_VALUE) < 0) { + // Use a fatal error because _Py_NewReference() cannot report + // the error to the caller. + Py_FatalError("_Py_hashtable_set() memory allocation failed"); } +} + + +static void +_PyRefchain_Remove(PyInterpreterState *interp, PyObject *obj) +{ + void *value = _Py_hashtable_steal(REFCHAIN(interp), obj); +#ifndef NDEBUG + assert(value == REFCHAIN_VALUE); +#else + (void)value; #endif - if (force || op->_ob_prev == NULL) { - PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); - op->_ob_next = refchain->_ob_next; - op->_ob_prev = refchain; - refchain->_ob_next->_ob_prev = op; - refchain->_ob_next = op; +} + + +/* Add an object to the refchain hash table. + * + * Note that objects are normally added to the list by PyObject_Init() + * indirectly. Not all objects are initialized that way, though; exceptions + * include statically allocated type objects, and statically allocated + * singletons (like Py_True and Py_None). */ +void +_Py_AddToAllObjects(PyObject *op) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!_PyRefchain_IsTraced(interp, op)) { + _PyRefchain_Trace(interp, op); } } #endif /* Py_TRACE_REFS */ @@ -471,16 +479,6 @@ _PyObject_IsFreed(PyObject *op) if (_PyMem_IsPtrFreed(op) || _PyMem_IsPtrFreed(Py_TYPE(op))) { return 1; } - /* ignore op->ob_ref: its value can have be modified - by Py_INCREF() and Py_DECREF(). */ -#ifdef Py_TRACE_REFS - if (op->_ob_next != NULL && _PyMem_IsPtrFreed(op->_ob_next)) { - return 1; - } - if (op->_ob_prev != NULL && _PyMem_IsPtrFreed(op->_ob_prev)) { - return 1; - } -#endif return 0; } @@ -1929,7 +1927,6 @@ PyTypeObject _PyNone_Type = { }; PyObject _Py_NoneStruct = { - _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, &_PyNone_Type }; @@ -2032,7 +2029,6 @@ PyTypeObject _PyNotImplemented_Type = { }; PyObject _Py_NotImplementedStruct = { - _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, &_PyNotImplemented_Type }; @@ -2042,12 +2038,30 @@ void _PyObject_InitState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS - if (!_Py_IsMainInterpreter(interp)) { - init_refchain(interp); + _Py_hashtable_allocator_t alloc = { + // Don't use default PyMem_Malloc() and PyMem_Free() which + // require the caller to hold the GIL. + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree, + }; + REFCHAIN(interp) = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, + NULL, NULL, &alloc); + if (REFCHAIN(interp) == NULL) { + Py_FatalError("_PyObject_InitState() memory allocation failure"); } #endif } +void +_PyObject_FiniState(PyInterpreterState *interp) +{ +#ifdef Py_TRACE_REFS + _Py_hashtable_destroy(REFCHAIN(interp)); + REFCHAIN(interp) = NULL; +#endif +} + extern PyTypeObject _PyAnextAwaitable_Type; extern PyTypeObject _PyLegacyEventHandler_Type; @@ -2230,7 +2244,7 @@ new_reference(PyObject *op) // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 op->ob_refcnt = 1; #ifdef Py_TRACE_REFS - _Py_AddToAllObjects(op, 1); + _Py_AddToAllObjects(op); #endif } @@ -2258,53 +2272,62 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); - if (op == refchain || - op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) - { - _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); - } + PyInterpreterState *interp = _PyInterpreterState_GET(); #ifdef SLOW_UNREF_CHECK - PyObject *p; - for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { - if (p == op) { - break; - } - } - if (p == refchain) { + if (!_PyRefchain_Get(interp, op)) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); } #endif - op->_ob_next->_ob_prev = op->_ob_prev; - op->_ob_prev->_ob_next = op->_ob_next; - op->_ob_next = op->_ob_prev = NULL; + _PyRefchain_Remove(interp, op); } +static int +_Py_PrintReference(_Py_hashtable_t *ht, + const void *key, const void *value, + void *user_data) +{ + PyObject *op = (PyObject*)key; + FILE *fp = (FILE *)user_data; + fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); + if (PyObject_Print(op, fp, 0) != 0) { + PyErr_Clear(); + } + putc('\n', fp); + return 0; +} + + /* Print all live objects. Because PyObject_Print is called, the * interpreter must be in a healthy state. */ void _Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { - PyObject *op; if (interp == NULL) { interp = _PyInterpreterState_Main(); } fprintf(fp, "Remaining objects:\n"); - PyObject *refchain = REFCHAIN(interp); - for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { - fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); - if (PyObject_Print(op, fp, 0) != 0) { - PyErr_Clear(); - } - putc('\n', fp); - } + _Py_hashtable_foreach(REFCHAIN(interp), _Py_PrintReference, fp); } + +static int +_Py_PrintReferenceAddress(_Py_hashtable_t *ht, + const void *key, const void *value, + void *user_data) +{ + PyObject *op = (PyObject*)key; + FILE *fp = (FILE *)user_data; + fprintf(fp, "%p [%zd] %s\n", + (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); + return 0; +} + + /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ @@ -2315,47 +2338,96 @@ _Py_PrintReferences(PyInterpreterState *interp, FILE *fp) void _Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { - PyObject *op; - PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) - fprintf(fp, "%p [%zd] %s\n", (void *)op, - Py_REFCNT(op), Py_TYPE(op)->tp_name); + _Py_hashtable_foreach(REFCHAIN(interp), _Py_PrintReferenceAddress, fp); } + +typedef struct { + PyObject *self; + PyObject *args; + PyObject *list; + PyObject *type; + Py_ssize_t limit; +} _Py_GetObjectsData; + +enum { + _PY_GETOBJECTS_IGNORE = 0, + _PY_GETOBJECTS_ERROR = 1, + _PY_GETOBJECTS_STOP = 2, +}; + +static int +_Py_GetObject(_Py_hashtable_t *ht, + const void *key, const void *value, + void *user_data) +{ + PyObject *op = (PyObject *)key; + _Py_GetObjectsData *data = user_data; + if (data->limit > 0) { + if (PyList_GET_SIZE(data->list) >= data->limit) { + return _PY_GETOBJECTS_STOP; + } + } + + if (op == data->self) { + return _PY_GETOBJECTS_IGNORE; + } + if (op == data->args) { + return _PY_GETOBJECTS_IGNORE; + } + if (op == data->list) { + return _PY_GETOBJECTS_IGNORE; + } + if (data->type != NULL) { + if (op == data->type) { + return _PY_GETOBJECTS_IGNORE; + } + if (!Py_IS_TYPE(op, (PyTypeObject *)data->type)) { + return _PY_GETOBJECTS_IGNORE; + } + } + + if (PyList_Append(data->list, op) < 0) { + return _PY_GETOBJECTS_ERROR; + } + return 0; +} + + /* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { - int i, n; - PyObject *t = NULL; - PyObject *res, *op; - PyInterpreterState *interp = _PyInterpreterState_GET(); + Py_ssize_t limit; + PyObject *type = NULL; + if (!PyArg_ParseTuple(args, "n|O", &limit, &type)) { + return NULL; + } - if (!PyArg_ParseTuple(args, "i|O", &n, &t)) + PyObject *list = PyList_New(0); + if (list == NULL) { return NULL; - PyObject *refchain = REFCHAIN(interp); - op = refchain->_ob_next; - res = PyList_New(0); - if (res == NULL) + } + + _Py_GetObjectsData data = { + .self = self, + .args = args, + .list = list, + .type = type, + .limit = limit, + }; + PyInterpreterState *interp = _PyInterpreterState_GET(); + int res = _Py_hashtable_foreach(REFCHAIN(interp), _Py_GetObject, &data); + if (res == _PY_GETOBJECTS_ERROR) { + Py_DECREF(list); return NULL; - for (i = 0; (n == 0 || i < n) && op != refchain; i++) { - while (op == self || op == args || op == res || op == t || - (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { - op = op->_ob_next; - if (op == refchain) - return res; - } - if (PyList_Append(res, op) < 0) { - Py_DECREF(res); - return NULL; - } - op = op->_ob_next; } - return res; + return list; } #undef REFCHAIN +#undef REFCHAIN_VALUE #endif /* Py_TRACE_REFS */ diff --git a/Objects/setobject.c b/Objects/setobject.c index 6051e57731c7a3..ae3f0b8d5e55d8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2548,7 +2548,6 @@ static PyTypeObject _PySetDummy_Type = { }; static PyObject _dummy_struct = { - _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, &_PySetDummy_Type }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 8cf654fb6f812d..5ffc52ae674c82 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -98,7 +98,6 @@ PyTypeObject PyEllipsis_Type = { }; PyObject _Py_EllipsisObject = { - _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, &PyEllipsis_Type }; diff --git a/Objects/structseq.c b/Objects/structseq.c index 6c07e6366293f9..0ca622edc2ba37 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -573,9 +573,10 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) Py_ssize_t n_members, n_unnamed_members; #ifdef Py_TRACE_REFS - /* if the type object was chained, unchain it first + /* if the type object was traced, remove it first before overwriting its storage */ - if (type->ob_base.ob_base._ob_next) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyRefchain_IsTraced(interp, (PyObject *)type)) { _Py_ForgetReference((PyObject *)type); } #endif diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 7ce3de4d58d0d3..67e059c3f74b77 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7508,7 +7508,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) * to get type objects into the doubly-linked list of all objects. * Still, not all type objects go through PyType_Ready. */ - _Py_AddToAllObjects((PyObject *)type, 0); + _Py_AddToAllObjects((PyObject *)type); #endif /* Initialize tp_dict: _PyType_IsReady() tests if tp_dict != NULL */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 30dd717b0fea29..971067e2d4fcc1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3110,7 +3110,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) * result, programs leaking references to None and False (etc) * couldn't be diagnosed by examining sys.getobjects(0). */ -#define ADD_TO_ALL(OBJECT) _Py_AddToAllObjects((PyObject *)(OBJECT), 0) +#define ADD_TO_ALL(OBJECT) _Py_AddToAllObjects((PyObject *)(OBJECT)) #else #define ADD_TO_ALL(OBJECT) (void)0 #endif diff --git a/Python/hashtable.c b/Python/hashtable.c index 4e22a1a5509eb5..8f5e8168ba1339 100644 --- a/Python/hashtable.c +++ b/Python/hashtable.c @@ -226,7 +226,6 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, void *value) assert(entry == NULL); #endif - entry = ht->alloc.malloc(sizeof(_Py_hashtable_entry_t)); if (entry == NULL) { /* memory allocation failed */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7d362af32cbc36..ee5d4981da51c8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1956,31 +1956,30 @@ Py_FinalizeEx(void) // XXX Ensure finalizer errors are handled properly. finalize_interp_clear(tstate); - finalize_interp_delete(tstate->interp); - -#ifdef Py_REF_DEBUG - if (show_ref_count) { - _PyDebug_PrintTotalRefs(); - } - _Py_FinalizeRefTotal(runtime); -#endif - _Py_FinalizeAllocatedBlocks(runtime); #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed - * above by _Py_PrintReferences. - */ - + * above by _Py_PrintReferences. */ if (dump_refs) { _Py_PrintReferenceAddresses(tstate->interp, stderr); } - if (dump_refs_fp != NULL) { _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ + + finalize_interp_delete(tstate->interp); + +#ifdef Py_REF_DEBUG + if (show_ref_count) { + _PyDebug_PrintTotalRefs(); + } + _Py_FinalizeRefTotal(runtime); +#endif + _Py_FinalizeAllocatedBlocks(runtime); + #ifdef WITH_PYMALLOC if (malloc_stats) { _PyObject_DebugMallocStats(stderr); diff --git a/Python/pystate.c b/Python/pystate.c index 01651d79f9acc0..4a8808f700eaa2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -674,6 +674,7 @@ init_interpreter(PyInterpreterState *interp, _obmalloc_pools_INIT(interp->obmalloc.pools); memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } + _PyObject_InitState(interp); _PyEval_InitState(interp, pending_lock); @@ -1001,6 +1002,9 @@ PyInterpreterState_Delete(PyInterpreterState *interp) if (interp->id_mutex != NULL) { PyThread_free_lock(interp->id_mutex); } + + _PyObject_FiniState(interp); + free_interpreter(interp); } diff --git a/configure b/configure index 57e3307266c0b1..7fe4aead29a732 100755 --- a/configure +++ b/configure @@ -23571,8 +23571,9 @@ SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFO { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $SOABI" >&5 printf "%s\n" "$SOABI" >&6; } -# Release and debug (Py_DEBUG) ABI are compatible, but not Py_TRACE_REFS ABI -if test "$Py_DEBUG" = 'true' -a "$with_trace_refs" != "yes"; then +# Release build, debug build (Py_DEBUG), and trace refs build (Py_TRACE_REFS) +# are ABI compatible +if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} @@ -29962,7 +29963,7 @@ printf %s "checking for stdlib extension module _testclinic_limited... " >&6; } if test "$py_cv_module__testclinic_limited" != "n/a" then : - if test "$TEST_MODULES" = yes -a "$with_trace_refs" = "no" + if test "$TEST_MODULES" = yes then : if true then : @@ -30267,7 +30268,7 @@ printf %s "checking for stdlib extension module xxlimited... " >&6; } if test "$py_cv_module_xxlimited" != "n/a" then : - if test "$with_trace_refs" = "no" + if true then : if test "$ac_cv_func_dlopen" = yes then : @@ -30305,7 +30306,7 @@ printf %s "checking for stdlib extension module xxlimited_35... " >&6; } if test "$py_cv_module_xxlimited_35" != "n/a" then : - if test "$with_trace_refs" = "no" + if true then : if test "$ac_cv_func_dlopen" = yes then : diff --git a/configure.ac b/configure.ac index 6fb6e110647036..5673b374353b47 100644 --- a/configure.ac +++ b/configure.ac @@ -5684,8 +5684,9 @@ AC_MSG_CHECKING([SOABI]) SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} AC_MSG_RESULT([$SOABI]) -# Release and debug (Py_DEBUG) ABI are compatible, but not Py_TRACE_REFS ABI -if test "$Py_DEBUG" = 'true' -a "$with_trace_refs" != "yes"; then +# Release build, debug build (Py_DEBUG), and trace refs build (Py_TRACE_REFS) +# are ABI compatible +if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS AC_SUBST([ALT_SOABI]) ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} @@ -7229,7 +7230,7 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes -a "$with_trace_refs" = "no"]) +PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) @@ -7241,10 +7242,9 @@ PY_STDLIB_MOD([_ctypes_test], [], [$LIBM]) dnl Limited API template modules. -dnl The limited C API is not compatible with the Py_TRACE_REFS macro. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_35], [], [test "$ac_cv_func_dlopen" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) From d48760b2f1e28dd3c1a35721939f400a8ab619b8 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 31 Aug 2023 12:45:12 -0700 Subject: [PATCH 429/598] gh-108682: [Enum] raise TypeError if super().__new__ called in custom __new__ (GH-108704) When overriding the `__new__` method of an enum, the underlying data type should be created directly; i.e. . member = object.__new__(cls) member = int.__new__(cls, value) member = str.__new__(cls, value) Calling `super().__new__()` finds the lookup version of `Enum.__new__`, and will now raise an exception when detected. --- Doc/howto/enum.rst | 23 +- Lib/enum.py | 7 + Lib/test/test_enum.py | 284 ++++++++++++++---- ...-08-30-20-10-28.gh-issue-108682.c2gzLQ.rst | 2 + 4 files changed, 260 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-30-20-10-28.gh-issue-108682.c2gzLQ.rst diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 4312b4c8140f5c..28749754a54dba 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -426,10 +426,17 @@ enumeration, with the exception of special methods (:meth:`__str__`, :meth:`__add__`, etc.), descriptors (methods are also descriptors), and variable names listed in :attr:`_ignore_`. -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then +Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__`, any value(s) given to the enum member will be passed into those methods. See `Planet`_ for an example. +.. note:: + + The :meth:`__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`__new__` which is used after + class creation for lookup of existing members. See :ref:`new-vs-init` for + more details. + Restricted Enum subclassing --------------------------- @@ -895,6 +902,8 @@ Some rules: :meth:`__str__` method has been reset to their data types' :meth:`__str__` method. +.. _new-vs-init: + When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------ @@ -927,6 +936,11 @@ want one of them to be the value:: >>> print(Coordinate(3)) Coordinate.VY +.. warning:: + + *Do not* call ``super().__new__()``, as the lookup-only ``__new__`` is the one + that is found; instead, use the data type directly. + Finer Points ^^^^^^^^^^^^ @@ -1353,6 +1367,13 @@ to handle any extra arguments:: members; it is then replaced by Enum's :meth:`__new__` which is used after class creation for lookup of existing members. +.. warning:: + + *Do not* call ``super().__new__()``, as the lookup-only ``__new__`` is the one + that is found; instead, use the data type directly -- e.g.:: + + obj = int.__new__(cls, value) + OrderedEnum ^^^^^^^^^^^ diff --git a/Lib/enum.py b/Lib/enum.py index 0c985b2c778569..4b99e7bda2cca5 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -856,6 +856,8 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s value = first_enum._generate_next_value_(name, start, count, last_values[:]) last_values.append(value) names.append((name, value)) + if names is None: + names = () # Here, names is either an iterable of (name, value) or a mapping. for item in names: @@ -1112,6 +1114,11 @@ def __new__(cls, value): for member in cls._member_map_.values(): if member._value_ == value: return member + # still not found -- verify that members exist, in-case somebody got here mistakenly + # (such as via super when trying to override __new__) + if not cls._member_map_: + raise TypeError("%r has no members defined" % cls) + # # still not found -- try _missing_ hook try: exc = None diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 36a1ee47640849..11a5b425efff9a 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -276,11 +276,82 @@ class _EnumTests: values = None def setUp(self): - class BaseEnum(self.enum_type): + if self.__class__.__name__[-5:] == 'Class': + class BaseEnum(self.enum_type): + @enum.property + def first(self): + return '%s is first!' % self.name + class MainEnum(BaseEnum): + first = auto() + second = auto() + third = auto() + if issubclass(self.enum_type, Flag): + dupe = 3 + else: + dupe = third + self.MainEnum = MainEnum + # + class NewStrEnum(self.enum_type): + def __str__(self): + return self.name.upper() + first = auto() + self.NewStrEnum = NewStrEnum + # + class NewFormatEnum(self.enum_type): + def __format__(self, spec): + return self.name.upper() + first = auto() + self.NewFormatEnum = NewFormatEnum + # + class NewStrFormatEnum(self.enum_type): + def __str__(self): + return self.name.title() + def __format__(self, spec): + return ''.join(reversed(self.name)) + first = auto() + self.NewStrFormatEnum = NewStrFormatEnum + # + class NewBaseEnum(self.enum_type): + def __str__(self): + return self.name.title() + def __format__(self, spec): + return ''.join(reversed(self.name)) + class NewSubEnum(NewBaseEnum): + first = auto() + self.NewSubEnum = NewSubEnum + # + class LazyGNV(self.enum_type): + def _generate_next_value_(name, start, last, values): + pass + self.LazyGNV = LazyGNV + # + class BusyGNV(self.enum_type): + @staticmethod + def _generate_next_value_(name, start, last, values): + pass + self.BusyGNV = BusyGNV + # + self.is_flag = False + self.names = ['first', 'second', 'third'] + if issubclass(MainEnum, StrEnum): + self.values = self.names + elif MainEnum._member_type_ is str: + self.values = ['1', '2', '3'] + elif issubclass(self.enum_type, Flag): + self.values = [1, 2, 4] + self.is_flag = True + self.dupe2 = MainEnum(5) + else: + self.values = self.values or [1, 2, 3] + # + if not getattr(self, 'source_values', False): + self.source_values = self.values + elif self.__class__.__name__[-8:] == 'Function': @enum.property def first(self): return '%s is first!' % self.name - class MainEnum(BaseEnum): + BaseEnum = self.enum_type('BaseEnum', {'first':first}) + # first = auto() second = auto() third = auto() @@ -288,63 +359,60 @@ class MainEnum(BaseEnum): dupe = 3 else: dupe = third - self.MainEnum = MainEnum - # - class NewStrEnum(self.enum_type): + self.MainEnum = MainEnum = BaseEnum('MainEnum', dict(first=first, second=second, third=third, dupe=dupe)) + # def __str__(self): return self.name.upper() first = auto() - self.NewStrEnum = NewStrEnum - # - class NewFormatEnum(self.enum_type): + self.NewStrEnum = self.enum_type('NewStrEnum', (('first',first),('__str__',__str__))) + # def __format__(self, spec): return self.name.upper() first = auto() - self.NewFormatEnum = NewFormatEnum - # - class NewStrFormatEnum(self.enum_type): + self.NewFormatEnum = self.enum_type('NewFormatEnum', [('first',first),('__format__',__format__)]) + # def __str__(self): return self.name.title() def __format__(self, spec): return ''.join(reversed(self.name)) first = auto() - self.NewStrFormatEnum = NewStrFormatEnum - # - class NewBaseEnum(self.enum_type): + self.NewStrFormatEnum = self.enum_type('NewStrFormatEnum', dict(first=first, __format__=__format__, __str__=__str__)) + # def __str__(self): return self.name.title() def __format__(self, spec): return ''.join(reversed(self.name)) - class NewSubEnum(NewBaseEnum): - first = auto() - self.NewSubEnum = NewSubEnum - # - class LazyGNV(self.enum_type): + NewBaseEnum = self.enum_type('NewBaseEnum', dict(__format__=__format__, __str__=__str__)) + class NewSubEnum(NewBaseEnum): + first = auto() + self.NewSubEnum = NewBaseEnum('NewSubEnum', 'first') + # def _generate_next_value_(name, start, last, values): pass - self.LazyGNV = LazyGNV - # - class BusyGNV(self.enum_type): + self.LazyGNV = self.enum_type('LazyGNV', {'_generate_next_value_':_generate_next_value_}) + # @staticmethod def _generate_next_value_(name, start, last, values): pass - self.BusyGNV = BusyGNV - # - self.is_flag = False - self.names = ['first', 'second', 'third'] - if issubclass(MainEnum, StrEnum): - self.values = self.names - elif MainEnum._member_type_ is str: - self.values = ['1', '2', '3'] - elif issubclass(self.enum_type, Flag): - self.values = [1, 2, 4] - self.is_flag = True - self.dupe2 = MainEnum(5) + self.BusyGNV = self.enum_type('BusyGNV', {'_generate_next_value_':_generate_next_value_}) + # + self.is_flag = False + self.names = ['first', 'second', 'third'] + if issubclass(MainEnum, StrEnum): + self.values = self.names + elif MainEnum._member_type_ is str: + self.values = ['1', '2', '3'] + elif issubclass(self.enum_type, Flag): + self.values = [1, 2, 4] + self.is_flag = True + self.dupe2 = MainEnum(5) + else: + self.values = self.values or [1, 2, 3] + # + if not getattr(self, 'source_values', False): + self.source_values = self.values else: - self.values = self.values or [1, 2, 3] - # - if not getattr(self, 'source_values', False): - self.source_values = self.values + raise ValueError('unknown enum style: %r' % self.__class__.__name__) def assertFormatIsValue(self, spec, member): self.assertEqual(spec.format(member), spec.format(member.value)) @@ -372,6 +440,17 @@ def spam(cls): with self.assertRaises(AttributeError): del Season.SPRING.name + def test_bad_new_super(self): + with self.assertRaisesRegex( + TypeError, + 'has no members defined', + ): + class BadSuper(self.enum_type): + def __new__(cls, value): + obj = super().__new__(cls, value) + return obj + failed = 1 + def test_basics(self): TE = self.MainEnum if self.is_flag: @@ -427,7 +506,7 @@ def test_contains_tf(self): MainEnum = self.MainEnum self.assertIn(MainEnum.first, MainEnum) self.assertTrue(self.values[0] in MainEnum) - if type(self) is not TestStrEnum: + if type(self) not in (TestStrEnumClass, TestStrEnumFunction): self.assertFalse('first' in MainEnum) val = MainEnum.dupe self.assertIn(val, MainEnum) @@ -949,15 +1028,23 @@ class OpenXYZ(self.enum_type): self.assertTrue(~OpenXYZ(0), (X|Y|Z)) -class TestPlainEnum(_EnumTests, _PlainOutputTests, unittest.TestCase): +class TestPlainEnumClass(_EnumTests, _PlainOutputTests, unittest.TestCase): + enum_type = Enum + + +class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): enum_type = Enum -class TestPlainFlag(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): +class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag -class TestIntEnum(_EnumTests, _MinimalOutputTests, unittest.TestCase): +class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): + enum_type = Flag + + +class TestIntEnumClass(_EnumTests, _MinimalOutputTests, unittest.TestCase): enum_type = IntEnum # def test_shadowed_attr(self): @@ -969,7 +1056,17 @@ class Number(IntEnum): self.assertIs(Number.numerator.divisor, Number.divisor) -class TestStrEnum(_EnumTests, _MinimalOutputTests, unittest.TestCase): +class TestIntEnumFunction(_EnumTests, _MinimalOutputTests, unittest.TestCase): + enum_type = IntEnum + # + def test_shadowed_attr(self): + Number = IntEnum('Number', ('divisor', 'numerator')) + # + self.assertEqual(Number.divisor.numerator, 1) + self.assertIs(Number.numerator.divisor, Number.divisor) + + +class TestStrEnumClass(_EnumTests, _MinimalOutputTests, unittest.TestCase): enum_type = StrEnum # def test_shadowed_attr(self): @@ -982,64 +1079,141 @@ class Book(StrEnum): self.assertIs(Book.title.author, Book.author) -class TestIntFlag(_EnumTests, _MinimalOutputTests, _FlagTests, unittest.TestCase): +class TestStrEnumFunction(_EnumTests, _MinimalOutputTests, unittest.TestCase): + enum_type = StrEnum + # + def test_shadowed_attr(self): + Book = StrEnum('Book', ('author', 'title')) + # + self.assertEqual(Book.author.title(), 'Author') + self.assertEqual(Book.title.title(), 'Title') + self.assertIs(Book.title.author, Book.author) + + +class TestIntFlagClass(_EnumTests, _MinimalOutputTests, _FlagTests, unittest.TestCase): + enum_type = IntFlag + + +class TestIntFlagFunction(_EnumTests, _MinimalOutputTests, _FlagTests, unittest.TestCase): enum_type = IntFlag -class TestMixedInt(_EnumTests, _MixedOutputTests, unittest.TestCase): +class TestMixedIntClass(_EnumTests, _MixedOutputTests, unittest.TestCase): class enum_type(int, Enum): pass -class TestMixedStr(_EnumTests, _MixedOutputTests, unittest.TestCase): +class TestMixedIntFunction(_EnumTests, _MixedOutputTests, unittest.TestCase): + enum_type = Enum('enum_type', type=int) + + +class TestMixedStrClass(_EnumTests, _MixedOutputTests, unittest.TestCase): class enum_type(str, Enum): pass -class TestMixedIntFlag(_EnumTests, _MixedOutputTests, _FlagTests, unittest.TestCase): +class TestMixedStrFunction(_EnumTests, _MixedOutputTests, unittest.TestCase): + enum_type = Enum('enum_type', type=str) + + +class TestMixedIntFlagClass(_EnumTests, _MixedOutputTests, _FlagTests, unittest.TestCase): class enum_type(int, Flag): pass -class TestMixedDate(_EnumTests, _MixedOutputTests, unittest.TestCase): +class TestMixedIntFlagFunction(_EnumTests, _MixedOutputTests, _FlagTests, unittest.TestCase): + enum_type = Flag('enum_type', type=int) + +class TestMixedDateClass(_EnumTests, _MixedOutputTests, unittest.TestCase): + # values = [date(2021, 12, 25), date(2020, 3, 15), date(2019, 11, 27)] source_values = [(2021, 12, 25), (2020, 3, 15), (2019, 11, 27)] - + # class enum_type(date, Enum): + @staticmethod def _generate_next_value_(name, start, count, last_values): values = [(2021, 12, 25), (2020, 3, 15), (2019, 11, 27)] return values[count] -class TestMinimalDate(_EnumTests, _MinimalOutputTests, unittest.TestCase): +class TestMixedDateFunction(_EnumTests, _MixedOutputTests, unittest.TestCase): + # + values = [date(2021, 12, 25), date(2020, 3, 15), date(2019, 11, 27)] + source_values = [(2021, 12, 25), (2020, 3, 15), (2019, 11, 27)] + # + # staticmethod decorator will be added by EnumType if not present + def _generate_next_value_(name, start, count, last_values): + values = [(2021, 12, 25), (2020, 3, 15), (2019, 11, 27)] + return values[count] + # + enum_type = Enum('enum_type', {'_generate_next_value_':_generate_next_value_}, type=date) + +class TestMinimalDateClass(_EnumTests, _MinimalOutputTests, unittest.TestCase): + # values = [date(2023, 12, 1), date(2016, 2, 29), date(2009, 1, 1)] source_values = [(2023, 12, 1), (2016, 2, 29), (2009, 1, 1)] - + # class enum_type(date, ReprEnum): + # staticmethod decorator will be added by EnumType if absent def _generate_next_value_(name, start, count, last_values): values = [(2023, 12, 1), (2016, 2, 29), (2009, 1, 1)] return values[count] -class TestMixedFloat(_EnumTests, _MixedOutputTests, unittest.TestCase): +class TestMinimalDateFunction(_EnumTests, _MinimalOutputTests, unittest.TestCase): + # + values = [date(2023, 12, 1), date(2016, 2, 29), date(2009, 1, 1)] + source_values = [(2023, 12, 1), (2016, 2, 29), (2009, 1, 1)] + # + @staticmethod + def _generate_next_value_(name, start, count, last_values): + values = [(2023, 12, 1), (2016, 2, 29), (2009, 1, 1)] + return values[count] + # + enum_type = ReprEnum('enum_type', {'_generate_next_value_':_generate_next_value_}, type=date) - values = [1.1, 2.2, 3.3] +class TestMixedFloatClass(_EnumTests, _MixedOutputTests, unittest.TestCase): + # + values = [1.1, 2.2, 3.3] + # class enum_type(float, Enum): def _generate_next_value_(name, start, count, last_values): values = [1.1, 2.2, 3.3] return values[count] -class TestMinimalFloat(_EnumTests, _MinimalOutputTests, unittest.TestCase): +class TestMixedFloatFunction(_EnumTests, _MixedOutputTests, unittest.TestCase): + # + values = [1.1, 2.2, 3.3] + # + def _generate_next_value_(name, start, count, last_values): + values = [1.1, 2.2, 3.3] + return values[count] + # + enum_type = Enum('enum_type', {'_generate_next_value_':_generate_next_value_}, type=float) - values = [4.4, 5.5, 6.6] +class TestMinimalFloatClass(_EnumTests, _MinimalOutputTests, unittest.TestCase): + # + values = [4.4, 5.5, 6.6] + # class enum_type(float, ReprEnum): def _generate_next_value_(name, start, count, last_values): values = [4.4, 5.5, 6.6] return values[count] +class TestMinimalFloatFunction(_EnumTests, _MinimalOutputTests, unittest.TestCase): + # + values = [4.4, 5.5, 6.6] + # + def _generate_next_value_(name, start, count, last_values): + values = [4.4, 5.5, 6.6] + return values[count] + # + enum_type = ReprEnum('enum_type', {'_generate_next_value_':_generate_next_value_}, type=float) + + class TestSpecial(unittest.TestCase): """ various operations that are not attributable to every possible enum diff --git a/Misc/NEWS.d/next/Library/2023-08-30-20-10-28.gh-issue-108682.c2gzLQ.rst b/Misc/NEWS.d/next/Library/2023-08-30-20-10-28.gh-issue-108682.c2gzLQ.rst new file mode 100644 index 00000000000000..148d4329142740 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-30-20-10-28.gh-issue-108682.c2gzLQ.rst @@ -0,0 +1,2 @@ +Enum: raise :exc:`TypeError` if ``super().__new__()`` is called from a +custom ``__new__``. From c6d56135e151a19c79d002cb1f1dbcd1c766c51d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 23:14:23 +0200 Subject: [PATCH 430/598] gh-108638: Fix tests when _stat extension is missing (#108689) Fix test_inspect and test_pydoc when the _stat extension is missing. Skip tests relying on _stat when _stat is missing. --- Lib/test/test_inspect.py | 28 ++++++++++++++++++++++------ Lib/test/test_pydoc.py | 8 +++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 9cb92c02d3e7d8..78ef817906b2aa 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1186,7 +1186,7 @@ def test_getfullargspec_builtin_func_no_signature(self): cls = _testcapi.DocStringNoSignatureTest obj = _testcapi.DocStringNoSignatureTest() - for builtin, template in [ + tests = [ (_testcapi.docstring_no_signature_noargs, meth_noargs), (_testcapi.docstring_no_signature_o, meth_o), (cls.meth_noargs, meth_self_noargs), @@ -1201,7 +1201,6 @@ def test_getfullargspec_builtin_func_no_signature(self): (cls.meth_o_coexist, meth_self_o), (time.time, meth_noargs), - (stat.S_IMODE, meth_o), (str.lower, meth_self_noargs), (''.lower, meth_self_noargs), (set.add, meth_self_o), @@ -1212,7 +1211,16 @@ def test_getfullargspec_builtin_func_no_signature(self): (datetime.datetime.utcnow, meth_type_noargs), (dict.__dict__['__class_getitem__'], meth_type_o), (dict.__class_getitem__, meth_type_o), - ]: + ] + try: + import _stat + except ImportError: + # if the _stat extension is not available, stat.S_IMODE() is + # implemented in Python, not in C + pass + else: + tests.append((stat.S_IMODE, meth_o)) + for builtin, template in tests: with self.subTest(builtin): self.assertEqual(inspect.getfullargspec(builtin), inspect.getfullargspec(template)) @@ -2934,7 +2942,7 @@ def test_signature_on_builtins_no_signature(self): cls = _testcapi.DocStringNoSignatureTest obj = _testcapi.DocStringNoSignatureTest() - for builtin, template in [ + tests = [ (_testcapi.docstring_no_signature_noargs, meth_noargs), (_testcapi.docstring_no_signature_o, meth_o), (cls.meth_noargs, meth_self_noargs), @@ -2949,7 +2957,6 @@ def test_signature_on_builtins_no_signature(self): (cls.meth_o_coexist, meth_self_o), (time.time, meth_noargs), - (stat.S_IMODE, meth_o), (str.lower, meth_self_noargs), (''.lower, meth_noargs), (set.add, meth_self_o), @@ -2960,7 +2967,16 @@ def test_signature_on_builtins_no_signature(self): (datetime.datetime.utcnow, meth_noargs), (dict.__dict__['__class_getitem__'], meth_type_o), (dict.__class_getitem__, meth_o), - ]: + ] + try: + import _stat + except ImportError: + # if the _stat extension is not available, stat.S_IMODE() is + # implemented in Python, not in C + pass + else: + tests.append((stat.S_IMODE, meth_o)) + for builtin, template in tests: with self.subTest(builtin): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 9b5c11bf853fd9..499eeb98ad6138 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1187,7 +1187,13 @@ def test_module_level_callable_noargs(self): "time()") def test_module_level_callable_o(self): - self.assertEqual(self._get_summary_line(stat.S_IMODE), + try: + import _stat + except ImportError: + # stat.S_IMODE() and _stat.S_IMODE() have a different signature + self.skipTest('_stat extension is missing') + + self.assertEqual(self._get_summary_line(_stat.S_IMODE), "S_IMODE(object, /)") def test_unbound_builtin_method_noargs(self): From 2bd960b57944107fbfbd8ff005b4223e1ea6555f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 23:41:18 +0200 Subject: [PATCH 431/598] gh-108337: Add pyatomic.h header (#108701) This adds a new header that provides atomic operations on common data types. The intention is that this will be exposed through Python.h, although that is not the case yet. The only immediate use is in the test file. Co-authored-by: Sam Gross --- Doc/whatsnew/3.13.rst | 3 + Include/cpython/pyatomic.h | 506 ++++++++++ Include/cpython/pyatomic_gcc.h | 499 +++++++++ Include/cpython/pyatomic_msc.h | 944 ++++++++++++++++++ Include/cpython/pyatomic_std.h | 872 ++++++++++++++++ Include/internal/pycore_atomic.h | 6 +- Lib/test/test_capi/test_pyatomic.py | 15 + Makefile.pre.in | 3 + ...-08-22-13-00-54.gh-issue-108337.wceHZm.rst | 1 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/parts.h | 1 + Modules/_testcapi/pyatomic.c | 180 ++++ Modules/_testcapimodule.c | 3 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 12 + 17 files changed, 3049 insertions(+), 4 deletions(-) create mode 100644 Include/cpython/pyatomic.h create mode 100644 Include/cpython/pyatomic_gcc.h create mode 100644 Include/cpython/pyatomic_msc.h create mode 100644 Include/cpython/pyatomic_std.h create mode 100644 Lib/test/test_capi/test_pyatomic.py create mode 100644 Misc/NEWS.d/next/C API/2023-08-22-13-00-54.gh-issue-108337.wceHZm.rst create mode 100644 Modules/_testcapi/pyatomic.c diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index fd2f2c3fff8282..d35d20ebb25293 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -833,6 +833,9 @@ Build Changes :ref:`debug build `. (Contributed by Victor Stinner in :gh:`108634`.) +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. + C API Changes ============= diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h new file mode 100644 index 00000000000000..73712db847087d --- /dev/null +++ b/Include/cpython/pyatomic.h @@ -0,0 +1,506 @@ +// This header provides cross-platform low-level atomic operations +// similar to C11 atomics. +// +// Operations are sequentially consistent unless they have a suffix indicating +// otherwise. If in doubt, prefer the sequentially consistent operations. +// +// The "_relaxed" suffix for load and store operations indicates the "relaxed" +// memory order. They don't provide synchronization, but (roughly speaking) +// guarantee somewhat sane behavior for races instead of undefined behavior. +// In practice, they correspond to "normal" hardware load and store +// instructions, so they are almost as inexpensive as plain loads and stores +// in C. +// +// Note that atomic read-modify-write operations like _Py_atomic_add_* return +// the previous value of the atomic variable, not the new value. +// +// See https://en.cppreference.com/w/c/atomic for more information on C11 +// atomics. +// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +// "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage +// or relaxed atomics. +// +// Functions with pseudo Python code: +// +// def _Py_atomic_load(obj): +// return obj # sequential consistency +// +// def _Py_atomic_load_relaxed(obj): +// return obj # relaxed consistency +// +// def _Py_atomic_store(obj, value): +// obj = value # sequential consistency +// +// def _Py_atomic_store_relaxed(obj, value): +// obj = value # relaxed consistency +// +// def _Py_atomic_exchange(obj, value): +// # sequential consistency +// old_obj = obj +// obj = value +// return old_obj +// +// def _Py_atomic_compare_exchange(obj, expected, desired): +// # sequential consistency +// if obj == expected: +// obj = desired +// return True +// else: +// expected = obj +// return False +// +// def _Py_atomic_add(obj, value): +// # sequential consistency +// old_obj = obj +// obj += value +// return old_obj +// +// def _Py_atomic_and(obj, value): +// # sequential consistency +// old_obj = obj +// obj &= value +// return old_obj +// +// def _Py_atomic_or(obj, value): +// # sequential consistency +// old_obj = obj +// obj |= value +// return old_obj +// +// Other functions: +// +// def _Py_atomic_load_ptr_acquire(obj): +// return obj # acquire +// +// def _Py_atomic_store_ptr_release(obj): +// return obj # release +// +// def _Py_atomic_fence_seq_cst(): +// # sequential consistency +// ... +// +// def _Py_atomic_fence_release(): +// # release +// ... + +#ifndef Py_ATOMIC_H +#define Py_ATOMIC_H + + +// --- _Py_atomic_add -------------------------------------------------------- +// Atomically adds `value` to `obj` and returns the previous value + +static inline int +_Py_atomic_add_int(int *obj, int value); + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value); + + +// --- _Py_atomic_compare_exchange ------------------------------------------- +// Performs an atomic compare-and-exchange. +// +// - If `*obj` and `*expected` are equal, store `desired` into `*obj` +// and return 1 (success). +// - Otherwise, store the `*obj` current value into `*expected` +// and return 0 (failure). +// +// These correspond to the C11 atomic_compare_exchange_strong() function. + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired); + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired); + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired); + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired); + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired); + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired); + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired); + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired); + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired); + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired); + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired); + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired); + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired); + +// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*` +// so that we can pass types like `PyObject**` without a cast. +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value); + + +// --- _Py_atomic_exchange --------------------------------------------------- +// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. + +static inline int +_Py_atomic_exchange_int(int *obj, int value); + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value); + +static inline void * +_Py_atomic_exchange_ptr(void *obj, void *value); + + +// --- _Py_atomic_and -------------------------------------------------------- +// Performs `*obj &= value` atomically and returns the previous value of `*obj`. + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value); + + +// --- _Py_atomic_or --------------------------------------------------------- +// Performs `*obj |= value` atomically and returns the previous value of `*obj`. + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value); + + +// --- _Py_atomic_load ------------------------------------------------------- +// Atomically loads `*obj` (sequential consistency) + +static inline int +_Py_atomic_load_int(const int *obj); + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj); + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj); + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj); + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj); + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj); + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj); + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj); + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj); + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj); + +static inline void * +_Py_atomic_load_ptr(const void *obj); + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- +// Loads `*obj` (relaxed consistency, i.e., no ordering) + +static inline int +_Py_atomic_load_int_relaxed(const int *obj); + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj); + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj); + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj); + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj); + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj); + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj); + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj); + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj); + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); + +static inline void * +_Py_atomic_load_ptr_relaxed(const void *obj); + + +// --- _Py_atomic_store ------------------------------------------------------ +// Atomically performs `*obj = value` (sequential consistency) + +static inline void +_Py_atomic_store_int(int *obj, int value); + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value); + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value); + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value); + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value); + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value); + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value); + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value); + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value); + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value); + +static inline void +_Py_atomic_store_ptr(void *obj, void *value); + +static inline void +_Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value); + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- +// Stores `*obj = value` (relaxed consistency, i.e., no ordering) + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value); + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value); + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value); + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value); + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value); + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value); + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value); + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value); + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value); + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value); + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value); + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +// Loads `*obj` (acquire operation) +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj); + +// Stores `*obj = value` (release operation) +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value); + + +// --- _Py_atomic_fence ------------------------------------------------------ + +// Sequential consistency fence. C11 fences have complex semantics. When +// possible, use the atomic operations on variables defined above, which +// generally do not require explicit use of a fence. +// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence +static inline void _Py_atomic_fence_seq_cst(void); + +// Release fence +static inline void _Py_atomic_fence_release(void); + + +#ifndef _Py_USE_GCC_BUILTIN_ATOMICS +# if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) +# define _Py_USE_GCC_BUILTIN_ATOMICS 1 +# elif defined(__clang__) +# if __has_builtin(__atomic_load) +# define _Py_USE_GCC_BUILTIN_ATOMICS 1 +# endif +# endif +#endif + +#if _Py_USE_GCC_BUILTIN_ATOMICS +# define Py_ATOMIC_GCC_H +# include "cpython/pyatomic_gcc.h" +# undef Py_ATOMIC_GCC_H +#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +# define Py_ATOMIC_STD_H +# include "cpython/pyatomic_std.h" +# undef Py_ATOMIC_STD_H +#elif defined(_MSC_VER) +# define Py_ATOMIC_MSC_H +# include "cpython/pyatomic_msc.h" +# undef Py_ATOMIC_MSC_H +#else +# error "no available pyatomic implementation for this platform/compiler" +#endif + +#endif /* Py_ATOMIC_H */ + diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h new file mode 100644 index 00000000000000..f1a38c7b52871a --- /dev/null +++ b/Include/cpython/pyatomic_gcc.h @@ -0,0 +1,499 @@ +// This is the implementation of Python atomic operations using GCC's built-in +// functions that match the C+11 memory model. This implementation is preferred +// for GCC compatible compilers, such as Clang. These functions are available +// in GCC 4.8+ without needing to compile with --std=c11 or --std=gnu11. + +#ifndef Py_ATOMIC_GCC_H +# error "this header file must not be included directly" +#endif + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) +{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void * +_Py_atomic_exchange_ptr(void *obj, void *value) +{ return __atomic_exchange_n((void **)obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline int +_Py_atomic_load_int(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline void * +_Py_atomic_load_ptr(const void *obj) +{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline void * +_Py_atomic_load_ptr_relaxed(const void *obj) +{ return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); } + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } + + +// --- _Py_atomic_fence ------------------------------------------------------ + +static inline void +_Py_atomic_fence_seq_cst(void) +{ __atomic_thread_fence(__ATOMIC_SEQ_CST); } + + static inline void +_Py_atomic_fence_release(void) +{ __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h new file mode 100644 index 00000000000000..c88bb03cc8f94a --- /dev/null +++ b/Include/cpython/pyatomic_msc.h @@ -0,0 +1,944 @@ +// This is the implementation of Python atomic operations for MSVC if the +// compiler does not support C11 or C++11 atomics. +// +// MSVC intrinsics are defined on char, short, long, __int64, and pointer +// types. Note that long and int are both 32-bits even on 64-bit Windows, +// so operations on int are cast to long. +// +// The volatile keyword has additional memory ordering semantics on MSVC. On +// x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64, +// volatile accesses behave like C11's memory_order_relaxed. + +#ifndef Py_ATOMIC_MSC_H +# error "this header file must not be included directly" +#endif + +#include + +#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \ + Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE)) + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(int64_t); + return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +#endif +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(__int64); + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value) +{ + void *initial = _InterlockedCompareExchangePointer( + (void**)obj, + value, + *(void**)expected); + if (initial == *(void**)expected) { + return 1; + } + *(void**)expected = initial; + return 0; +} + + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value) +{ + return _Py_atomic_compare_exchange_int8((int8_t *)obj, + (int8_t *)expected, + (int8_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value) +{ + return _Py_atomic_compare_exchange_int16((int16_t *)obj, + (int16_t *)expected, + (int16_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value) +{ + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value) +{ + return _Py_atomic_compare_exchange_int64((int64_t *)obj, + (int64_t *)expected, + (int64_t)value); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) { + return old_value; + } + } +#endif +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value); +} + + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj, + (int8_t)value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj, + (int16_t)value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj, + (int64_t)value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value & value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value | value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint8_t *)obj; +#elif defined(_M_ARM64) + return (uint8_t)__ldar8((unsigned __int8 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint8" +#endif +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint16_t *)obj; +#elif defined(_M_ARM64) + return (uint16_t)__ldar16((unsigned __int16 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint16" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint32_t *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint64_t *)obj; +#elif defined(_M_ARM64) + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64" +#endif +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj); +} + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ +#if SIZEOF_VOID_P == 8 + return (void*)_Py_atomic_load_uint64((const uint64_t *)obj); +#else + return (void*)_Py_atomic_load_uint32((const uint32_t *)obj); +#endif +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + return *(volatile int *)obj; +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + return *(volatile int8_t *)obj; +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + return *(volatile int16_t *)obj; +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + return *(volatile int32_t *)obj; +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + return *(volatile int64_t *)obj; +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + return *(volatile intptr_t *)obj; +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + return *(volatile uint8_t *)obj; +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + return *(volatile uint16_t *)obj; +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + return *(volatile uint32_t *)obj; +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + return *(volatile uint64_t *)obj; +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + return *(volatile uintptr_t *)obj; +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + return *(volatile unsigned int *)obj; +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + return *(volatile Py_ssize_t *)obj; +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + return *(void * volatile *)obj; +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + (void)_Py_atomic_exchange_int(obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + (void)_Py_atomic_exchange_int8(obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + (void)_Py_atomic_exchange_int16(obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + (void)_Py_atomic_exchange_int32(obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + (void)_Py_atomic_exchange_int64(obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + (void)_Py_atomic_exchange_intptr(obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + (void)_Py_atomic_exchange_uint8(obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + (void)_Py_atomic_exchange_uint16(obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + (void)_Py_atomic_exchange_uint32(obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + (void)_Py_atomic_exchange_uint64(obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + (void)_Py_atomic_exchange_uintptr(obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + (void)_Py_atomic_exchange_uint(obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + (void)_Py_atomic_exchange_ptr(obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + (void)_Py_atomic_exchange_ssize(obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + *(volatile int *)obj = value; +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + *(volatile int8_t *)obj = value; +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + *(volatile int16_t *)obj = value; +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + *(volatile int32_t *)obj = value; +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + *(volatile int64_t *)obj = value; +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + *(volatile intptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + *(volatile uint8_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + *(volatile uint16_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + *(volatile uint32_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + *(volatile uint64_t *)obj = value; +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + *(volatile uintptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void* value) +{ + *(void * volatile *)obj = value; +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + *(volatile Py_ssize_t *)obj = value; +} + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(void * volatile *)obj; +#elif defined(_M_ARM64) + return (void *)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ptr_acquire" +#endif +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(void * volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64(obj, (uintptr_t)value); +#else +# error "no implementation of _Py_atomic_store_ptr_release" +#endif +} + + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) + __faststorefence(); +#elif defined(_M_IX86) + _mm_mfence(); +#else +# error "no implementation of _Py_atomic_fence_seq_cst" +#endif +} + + static inline void +_Py_atomic_fence_release(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadWriteBarrier(); +#else +# error "no implementation of _Py_atomic_fence_release" +#endif +} + +#undef _Py_atomic_ASSERT_ARG_TYPE diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h new file mode 100644 index 00000000000000..bf74a90887c634 --- /dev/null +++ b/Include/cpython/pyatomic_std.h @@ -0,0 +1,872 @@ +// This is the implementation of Python atomic operations using C++11 or C11 +// atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC +// compatible compilers, even if they support C++11 atomics. + +#ifndef Py_ATOMIC_STD_H +# error "this header file must not be included directly" +#endif + +#ifdef __cplusplus +extern "C++" { +# include +} +# define _Py_USING_STD using namespace std +# define _Atomic(tp) atomic +#else +# define _Py_USING_STD +# include +#endif + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int)*)obj, value); +} + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int8_t)*)obj, value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int16_t)*)obj, value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int32_t)*)obj, value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int64_t)*)obj, value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(intptr_t)*)obj, value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(unsigned int)*)obj, value); +} + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(void *)*)obj, + (void **)expected, desired); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int)*)obj, value); +} + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int8_t)*)obj, value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int16_t)*)obj, value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int32_t)*)obj, value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int64_t)*)obj, value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(intptr_t)*)obj, value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(unsigned int)*)obj, value); +} + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uintptr_t)*)obj, value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value); +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(void *)*)obj, value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value); +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value); +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int)*)obj); +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int8_t)*)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int16_t)*)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int32_t)*)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int64_t)*)obj); +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(intptr_t)*)obj); +} + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint8_t)*)obj); +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint32_t)*)obj); +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint32_t)*)obj); +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint64_t)*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uintptr_t)*)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(unsigned int)*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(Py_ssize_t)*)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(void*)*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_relaxed); +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int8_t)*)obj, + memory_order_relaxed); +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int16_t)*)obj, + memory_order_relaxed); +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int32_t)*)obj, + memory_order_relaxed); +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int64_t)*)obj, + memory_order_relaxed); +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(intptr_t)*)obj, + memory_order_relaxed); +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint8_t)*)obj, + memory_order_relaxed); +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint16_t)*)obj, + memory_order_relaxed); +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_relaxed); +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_relaxed); +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_relaxed); +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned int)*)obj, + memory_order_relaxed); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_relaxed); +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_relaxed); +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int)*)obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int8_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int16_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int32_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int64_t)*)obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(intptr_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint8_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint16_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint32_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint64_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uintptr_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store((_Atomic(unsigned int)*)obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store((_Atomic(void*)*)obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(Py_ssize_t)*)obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int16_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int32_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int64_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(intptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint8_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint16_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(void*)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_relaxed); +} + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_acquire); +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(void*)*)obj, value, + memory_order_release); +} + + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_seq_cst); +} + + static inline void +_Py_atomic_fence_release(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_release); +} diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h index 48d246ea08f3d9..22ce971a64f3df 100644 --- a/Include/internal/pycore_atomic.h +++ b/Include/internal/pycore_atomic.h @@ -1,5 +1,5 @@ -#ifndef Py_ATOMIC_H -#define Py_ATOMIC_H +#ifndef Py_INTERNAL_ATOMIC_H +#define Py_INTERNAL_ATOMIC_H #ifdef __cplusplus extern "C" { #endif @@ -554,4 +554,4 @@ typedef struct _Py_atomic_int { #ifdef __cplusplus } #endif -#endif /* Py_ATOMIC_H */ +#endif /* Py_INTERNAL_ATOMIC_H */ diff --git a/Lib/test/test_capi/test_pyatomic.py b/Lib/test/test_capi/test_pyatomic.py new file mode 100644 index 00000000000000..846d6d50c25969 --- /dev/null +++ b/Lib/test/test_capi/test_pyatomic.py @@ -0,0 +1,15 @@ +import unittest +from test.support import import_helper + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + +class PyAtomicTests(unittest.TestCase): + pass + +for name in sorted(dir(_testcapi)): + if name.startswith('test_atomic'): + setattr(PyAtomicTests, name, getattr(_testcapi, name)) + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index ad3e26c036408e..0870cb3d899d45 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1713,6 +1713,9 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/optimizer.h \ $(srcdir)/Include/cpython/picklebufobject.h \ $(srcdir)/Include/cpython/pthread_stubs.h \ + $(srcdir)/Include/cpython/pyatomic.h \ + $(srcdir)/Include/cpython/pyatomic_gcc.h \ + $(srcdir)/Include/cpython/pyatomic_std.h \ $(srcdir)/Include/cpython/pyctype.h \ $(srcdir)/Include/cpython/pydebug.h \ $(srcdir)/Include/cpython/pyerrors.h \ diff --git a/Misc/NEWS.d/next/C API/2023-08-22-13-00-54.gh-issue-108337.wceHZm.rst b/Misc/NEWS.d/next/C API/2023-08-22-13-00-54.gh-issue-108337.wceHZm.rst new file mode 100644 index 00000000000000..476123a051bb3f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-22-13-00-54.gh-issue-108337.wceHZm.rst @@ -0,0 +1 @@ +Add atomic operations on additional data types in pyatomic.h. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 7b9dc3c5c8e26e..2bb39d2936894a 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -159,7 +159,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 9c6d6157141801..c162dbc65db81a 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -22,6 +22,7 @@ int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); +int _PyTestCapi_Init_PyAtomic(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *mod); diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c new file mode 100644 index 00000000000000..15602ce3f4ab3b --- /dev/null +++ b/Modules/_testcapi/pyatomic.c @@ -0,0 +1,180 @@ +/* + * C Extension module to smoke test pyatomic.h API. + * + * This only tests basic functionality, not any synchronizing ordering. + */ + +/* Always enable assertions */ +#undef NDEBUG + +#include "Python.h" +#include "cpython/pyatomic.h" +#include "parts.h" + +// We define atomic bitwise operations on these types +#define FOR_BITWISE_TYPES(V) \ + V(uint8, uint8_t) \ + V(uint16, uint16_t) \ + V(uint32, uint32_t) \ + V(uint64, uint64_t) \ + V(uintptr, uintptr_t) + +// We define atomic addition on these types +#define FOR_ARITHMETIC_TYPES(V) \ + FOR_BITWISE_TYPES(V) \ + V(int, int) \ + V(uint, unsigned int) \ + V(int8, int8_t) \ + V(int16, int16_t) \ + V(int32, int32_t) \ + V(int64, int64_t) \ + V(intptr, intptr_t) \ + V(ssize, Py_ssize_t) + +// We define atomic load, store, exchange, and compare_exchange on these types +#define FOR_ALL_TYPES(V) \ + FOR_ARITHMETIC_TYPES(V) \ + V(ptr, void*) + +#define IMPL_TEST_ADD(suffix, dtype) \ +static PyObject * \ +test_atomic_add_##suffix(PyObject *self, PyObject *obj) { \ + dtype x = 0; \ + assert(_Py_atomic_add_##suffix(&x, 1) == 0); \ + assert(x == 1); \ + assert(_Py_atomic_add_##suffix(&x, 2) == 1); \ + assert(x == 3); \ + assert(_Py_atomic_add_##suffix(&x, -2) == 3); \ + assert(x == 1); \ + assert(_Py_atomic_add_##suffix(&x, -1) == 1); \ + assert(x == 0); \ + assert(_Py_atomic_add_##suffix(&x, -1) == 0); \ + assert(x == (dtype)-1); \ + assert(_Py_atomic_add_##suffix(&x, -2) == (dtype)-1); \ + assert(x == (dtype)-3); \ + assert(_Py_atomic_add_##suffix(&x, 2) == (dtype)-3); \ + assert(x == (dtype)-1); \ + Py_RETURN_NONE; \ +} +FOR_ARITHMETIC_TYPES(IMPL_TEST_ADD) + +#define IMPL_TEST_COMPARE_EXCHANGE(suffix, dtype) \ +static PyObject * \ +test_atomic_compare_exchange_##suffix(PyObject *self, PyObject *obj) { \ + dtype x = (dtype)0; \ + dtype y = (dtype)1; \ + dtype z = (dtype)2; \ + assert(_Py_atomic_compare_exchange_##suffix(&x, &y, z) == 0); \ + assert(x == 0); \ + assert(y == 0); \ + assert(_Py_atomic_compare_exchange_##suffix(&x, &y, z) == 1); \ + assert(x == z); \ + assert(y == 0); \ + assert(_Py_atomic_compare_exchange_##suffix(&x, &y, z) == 0); \ + assert(x == z); \ + assert(y == z); \ + Py_RETURN_NONE; \ +} +FOR_ALL_TYPES(IMPL_TEST_COMPARE_EXCHANGE) + +#define IMPL_TEST_EXCHANGE(suffix, dtype) \ +static PyObject * \ +test_atomic_exchange_##suffix(PyObject *self, PyObject *obj) { \ + dtype x = (dtype)0; \ + dtype y = (dtype)1; \ + dtype z = (dtype)2; \ + assert(_Py_atomic_exchange_##suffix(&x, y) == (dtype)0); \ + assert(x == (dtype)1); \ + assert(_Py_atomic_exchange_##suffix(&x, z) == (dtype)1); \ + assert(x == (dtype)2); \ + assert(_Py_atomic_exchange_##suffix(&x, y) == (dtype)2); \ + assert(x == (dtype)1); \ + Py_RETURN_NONE; \ +} +FOR_ALL_TYPES(IMPL_TEST_EXCHANGE) + +#define IMPL_TEST_LOAD_STORE(suffix, dtype) \ +static PyObject * \ +test_atomic_load_store_##suffix(PyObject *self, PyObject *obj) { \ + dtype x = (dtype)0; \ + dtype y = (dtype)1; \ + dtype z = (dtype)2; \ + assert(_Py_atomic_load_##suffix(&x) == (dtype)0); \ + assert(x == (dtype)0); \ + _Py_atomic_store_##suffix(&x, y); \ + assert(_Py_atomic_load_##suffix(&x) == (dtype)1); \ + assert(x == (dtype)1); \ + _Py_atomic_store_##suffix##_relaxed(&x, z); \ + assert(_Py_atomic_load_##suffix##_relaxed(&x) == (dtype)2); \ + assert(x == (dtype)2); \ + Py_RETURN_NONE; \ +} +FOR_ALL_TYPES(IMPL_TEST_LOAD_STORE) + +#define IMPL_TEST_AND_OR(suffix, dtype) \ +static PyObject * \ +test_atomic_and_or_##suffix(PyObject *self, PyObject *obj) { \ + dtype x = (dtype)0; \ + dtype y = (dtype)1; \ + dtype z = (dtype)3; \ + assert(_Py_atomic_or_##suffix(&x, z) == (dtype)0); \ + assert(x == (dtype)3); \ + assert(_Py_atomic_and_##suffix(&x, y) == (dtype)3); \ + assert(x == (dtype)1); \ + Py_RETURN_NONE; \ +} +FOR_BITWISE_TYPES(IMPL_TEST_AND_OR) + +static PyObject * +test_atomic_fences(PyObject *self, PyObject *obj) { + // Just make sure that the fences compile. We are not + // testing any synchronizing ordering. + _Py_atomic_fence_seq_cst(); + _Py_atomic_fence_release(); + Py_RETURN_NONE; +} + +static PyObject * +test_atomic_release_acquire(PyObject *self, PyObject *obj) { + void *x = NULL; + void *y = &y; + assert(_Py_atomic_load_ptr_acquire(&x) == NULL); + _Py_atomic_store_ptr_release(&x, y); + assert(x == y); + assert(_Py_atomic_load_ptr_acquire(&x) == y); + Py_RETURN_NONE; +} + +// NOTE: all tests should start with "test_atomic_" to be included +// in test_pyatomic.py + +#define BIND_TEST_ADD(suffix, dtype) \ + {"test_atomic_add_" #suffix, test_atomic_add_##suffix, METH_NOARGS}, +#define BIND_TEST_COMPARE_EXCHANGE(suffix, dtype) \ + {"test_atomic_compare_exchange_" #suffix, test_atomic_compare_exchange_##suffix, METH_NOARGS}, +#define BIND_TEST_EXCHANGE(suffix, dtype) \ + {"test_atomic_exchange_" #suffix, test_atomic_exchange_##suffix, METH_NOARGS}, +#define BIND_TEST_LOAD_STORE(suffix, dtype) \ + {"test_atomic_load_store_" #suffix, test_atomic_load_store_##suffix, METH_NOARGS}, +#define BIND_TEST_AND_OR(suffix, dtype) \ + {"test_atomic_and_or_" #suffix, test_atomic_and_or_##suffix, METH_NOARGS}, + +static PyMethodDef test_methods[] = { + FOR_ARITHMETIC_TYPES(BIND_TEST_ADD) + FOR_ALL_TYPES(BIND_TEST_COMPARE_EXCHANGE) + FOR_ALL_TYPES(BIND_TEST_EXCHANGE) + FOR_ALL_TYPES(BIND_TEST_LOAD_STORE) + FOR_BITWISE_TYPES(BIND_TEST_AND_OR) + {"test_atomic_fences", test_atomic_fences, METH_NOARGS}, + {"test_atomic_release_acquire", test_atomic_release_acquire, METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + +int +_PyTestCapi_Init_PyAtomic(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index d5c8a9d7ae520d..f8b36ff9ce835a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4004,6 +4004,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_GC(m) < 0) { return NULL; } + if (_PyTestCapi_Init_PyAtomic(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 8c0fd0cf052b0e..0a02929db438b8 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -112,6 +112,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 87d33ebe28e475..4ba6011d8af5b9 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -66,6 +66,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 0b0f5ad0fdb9cd..4cd095b28e510f 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -166,6 +166,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index c70b8017935d73..af1669209a9049 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -423,6 +423,18 @@ Include + + Include + + + Include + + + Include + + + Include + Include From ad73674283978a8bf8f1983bfd69150e60e6225f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Aug 2023 23:42:34 +0200 Subject: [PATCH 432/598] gh-107603: Argument Clinic: Only include pycore_gc.h if needed (#108726) Argument Clinic now only includes pycore_gc.h if PyGC_Head is needed, and only includes pycore_runtime.h if _Py_ID() is needed. * Add 'condition' optional argument to Clinic.add_include(). * deprecate_keyword_use() includes pycore_runtime.h when using the _PyID() function. * Fix rendering of includes: comments start at the column 35. * Mark PC/clinic/_wmimodule.cpp.h and "Objects/stringlib/clinic/*.h.h" header files as generated in .gitattributes. Effects: * 42 header files generated by AC no longer include the internal C API, instead of 4 header files before. For example, Modules/clinic/_abc.c.h no longer includes the internal C API. * Fix _testclinic_depr.c.h: it now always includes pycore_runtime.h to get _Py_ID(). --- .gitattributes | 2 + Lib/test/test_clinic.py | 8 +- Modules/_blake2/clinic/blake2b_impl.c.h | 9 +- Modules/_blake2/clinic/blake2s_impl.c.h | 9 +- Modules/_ctypes/clinic/callproc.c.h | 8 +- Modules/_io/clinic/_iomodule.c.h | 7 +- Modules/_io/clinic/bufferedio.c.h | 9 +- Modules/_io/clinic/bytesio.c.h | 9 +- Modules/_io/clinic/fileio.c.h | 9 +- Modules/_io/clinic/iobase.c.h | 9 +- Modules/_io/clinic/stringio.c.h | 9 +- Modules/_io/clinic/textio.c.h | 9 +- Modules/_io/clinic/winconsoleio.c.h | 9 +- .../clinic/multiprocessing.c.h | 8 +- .../_multiprocessing/clinic/posixshmem.c.h | 7 +- Modules/_multiprocessing/clinic/semaphore.c.h | 7 +- Modules/_sqlite/clinic/_sqlite3.connect.c.h | 7 +- Modules/_sqlite/clinic/blob.c.h | 8 +- Modules/_sqlite/clinic/connection.c.h | 7 +- Modules/_sqlite/clinic/cursor.c.h | 7 +- Modules/_sqlite/clinic/module.c.h | 7 +- Modules/_sqlite/clinic/row.c.h | 8 +- Modules/_sre/clinic/sre.c.h | 9 +- Modules/_ssl/clinic/cert.c.h | 7 +- Modules/_testcapi/clinic/exceptions.c.h | 7 +- Modules/_testcapi/clinic/float.c.h | 8 +- Modules/_testcapi/clinic/long.c.h | 8 +- Modules/_testcapi/clinic/vectorcall.c.h | 8 +- Modules/_testcapi/clinic/watchers.c.h | 8 +- Modules/cjkcodecs/clinic/multibytecodec.c.h | 7 +- Modules/clinic/_abc.c.h | 8 +- Modules/clinic/_asynciomodule.c.h | 7 +- Modules/clinic/_bisectmodule.c.h | 9 +- Modules/clinic/_bz2module.c.h | 9 +- Modules/clinic/_codecsmodule.c.h | 7 +- Modules/clinic/_collectionsmodule.c.h | 9 +- Modules/clinic/_contextvarsmodule.c.h | 8 +- Modules/clinic/_csv.c.h | 7 +- Modules/clinic/_curses_panel.c.h | 8 +- Modules/clinic/_cursesmodule.c.h | 7 +- Modules/clinic/_datetimemodule.c.h | 7 +- Modules/clinic/_dbmmodule.c.h | 8 +- Modules/clinic/_elementtree.c.h | 9 +- Modules/clinic/_functoolsmodule.c.h | 7 +- Modules/clinic/_gdbmmodule.c.h | 8 +- Modules/clinic/_hashopenssl.c.h | 9 +- Modules/clinic/_heapqmodule.c.h | 8 +- Modules/clinic/_localemodule.c.h | 8 +- Modules/clinic/_lsprof.c.h | 8 +- Modules/clinic/_lzmamodule.c.h | 9 +- Modules/clinic/_opcode.c.h | 7 +- Modules/clinic/_operator.c.h | 9 +- Modules/clinic/_pickle.c.h | 7 +- Modules/clinic/_posixsubprocess.c.h | 8 +- Modules/clinic/_queuemodule.c.h | 7 +- Modules/clinic/_randommodule.c.h | 8 +- Modules/clinic/_ssl.c.h | 9 +- Modules/clinic/_statisticsmodule.c.h | 8 +- Modules/clinic/_struct.c.h | 9 +- Modules/clinic/_testclinic.c.h | 11 +- Modules/clinic/_testclinic_depr.c.h | 11 +- Modules/clinic/_testinternalcapi.c.h | 7 +- Modules/clinic/_testmultiphase.c.h | 7 +- Modules/clinic/_tkinter.c.h | 8 +- Modules/clinic/_tracemalloc.c.h | 8 +- Modules/clinic/_typingmodule.c.h | 8 +- Modules/clinic/_weakref.c.h | 8 +- Modules/clinic/_winapi.c.h | 9 +- Modules/clinic/_zoneinfo.c.h | 7 +- Modules/clinic/arraymodule.c.h | 9 +- Modules/clinic/binascii.c.h | 7 +- Modules/clinic/cmathmodule.c.h | 7 +- Modules/clinic/fcntlmodule.c.h | 9 +- Modules/clinic/gcmodule.c.h | 9 +- Modules/clinic/grpmodule.c.h | 7 +- Modules/clinic/itertoolsmodule.c.h | 9 +- Modules/clinic/mathmodule.c.h | 7 +- Modules/clinic/md5module.c.h | 7 +- Modules/clinic/overlapped.c.h | 9 +- Modules/clinic/posixmodule.c.h | 13 +- Modules/clinic/pwdmodule.c.h | 8 +- Modules/clinic/pyexpat.c.h | 7 +- Modules/clinic/readline.c.h | 8 +- Modules/clinic/resource.c.h | 8 +- Modules/clinic/selectmodule.c.h | 11 +- Modules/clinic/sha1module.c.h | 7 +- Modules/clinic/sha2module.c.h | 7 +- Modules/clinic/sha3module.c.h | 9 +- Modules/clinic/signalmodule.c.h | 8 +- Modules/clinic/socketmodule.c.h | 7 +- Modules/clinic/symtablemodule.c.h | 8 +- Modules/clinic/syslogmodule.c.h | 7 +- Modules/clinic/termios.c.h | 9 +- Modules/clinic/unicodedata.c.h | 8 +- Modules/clinic/zlibmodule.c.h | 9 +- Objects/clinic/bytearrayobject.c.h | 9 +- Objects/clinic/bytesobject.c.h | 9 +- Objects/clinic/classobject.c.h | 8 +- Objects/clinic/codeobject.c.h | 7 +- Objects/clinic/complexobject.c.h | 7 +- Objects/clinic/descrobject.c.h | 7 +- Objects/clinic/dictobject.c.h | 8 +- Objects/clinic/enumobject.c.h | 7 +- Objects/clinic/floatobject.c.h | 8 +- Objects/clinic/funcobject.c.h | 7 +- Objects/clinic/listobject.c.h | 9 +- Objects/clinic/longobject.c.h | 9 +- Objects/clinic/memoryobject.c.h | 7 +- Objects/clinic/moduleobject.c.h | 7 +- Objects/clinic/odictobject.c.h | 7 +- Objects/clinic/structseq.c.h | 7 +- Objects/clinic/tupleobject.c.h | 8 +- Objects/clinic/typeobject.c.h | 8 +- Objects/clinic/typevarobject.c.h | 7 +- Objects/clinic/unicodeobject.c.h | 9 +- Objects/stringlib/clinic/transmogrify.h.h | 9 +- PC/clinic/_testconsole.c.h | 7 +- PC/clinic/_wmimodule.cpp.h | 7 +- PC/clinic/msvcrtmodule.c.h | 8 +- PC/clinic/winreg.c.h | 9 +- PC/clinic/winsound.c.h | 7 +- Python/clinic/Python-tokenize.c.h | 7 +- Python/clinic/_warnings.c.h | 9 +- Python/clinic/bltinmodule.c.h | 7 +- Python/clinic/context.c.h | 8 +- Python/clinic/import.c.h | 7 +- Python/clinic/instrumentation.c.h | 8 +- Python/clinic/marshal.c.h | 8 +- Python/clinic/sysmodule.c.h | 7 +- Python/clinic/traceback.c.h | 7 +- Tools/clinic/clinic.py | 141 +++++++++++++----- 131 files changed, 448 insertions(+), 727 deletions(-) diff --git a/.gitattributes b/.gitattributes index 317a613217a9b3..e05ff900bf1c09 100644 --- a/.gitattributes +++ b/.gitattributes @@ -66,6 +66,8 @@ PCbuild/readme.txt dos [attr]generated linguist-generated=true diff=generated **/clinic/*.c.h generated +**/clinic/*.cpp.h generated +**/clinic/*.h.h generated *_db.h generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index b29f2ea55516fe..4a5f472e6f7109 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2673,12 +2673,6 @@ def test_file_dest(self): preserve [clinic start generated code]*/ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # include "pycore_gc.h" // PyGC_Head - # include "pycore_runtime.h" // _Py_ID() - #endif - - PyDoc_VAR(func__doc__); PyDoc_STRVAR(func__doc__, @@ -2691,7 +2685,7 @@ def test_file_dest(self): static PyObject * func(PyObject *module, PyObject *a) - /*[clinic end generated code: output=56c09670e89a0d9a input=a9049054013a1b77]*/ + /*[clinic end generated code: output=3dde2d13002165b9 input=a9049054013a1b77]*/ """) with os_helper.temp_dir() as tmp_dir: in_fn = os.path.join(tmp_dir, "test.c") diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 384e7142bcfda7..869cbea78dd267 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(py_blake2b_new__doc__, "blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" @@ -277,4 +276,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=48128782266b7b8e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2ad807e0c83d8c25 input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index d7a475277e0543..affc203486b174 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(py_blake2s_new__doc__, "blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" @@ -277,4 +276,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=2188af9910a45497 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90ca2b52b8c40785 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/clinic/callproc.c.h b/Modules/_ctypes/clinic/callproc.c.h index 6f036bb66b25aa..a787693ae67cd8 100644 --- a/Modules/_ctypes/clinic/callproc.c.h +++ b/Modules/_ctypes/clinic/callproc.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(create_pointer_type__doc__, "POINTER($module, type, /)\n" "--\n" @@ -35,4 +29,4 @@ PyDoc_STRVAR(create_pointer_inst__doc__, #define CREATE_POINTER_INST_METHODDEF \ {"pointer", (PyCFunction)create_pointer_inst, METH_O, create_pointer_inst__doc__}, -/*[clinic end generated code: output=ae26452a759ba56d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=51b311ea369e5adf input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 77c203681fca44..49a3573ab97473 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_io_open__doc__, "open($module, /, file, mode=\'r\', buffering=-1, encoding=None,\n" " errors=None, newline=None, closefd=True, opener=None)\n" @@ -404,4 +403,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=a4ceb802f3a7243f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aaf96c8d9bd20abc input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 32c4a4e177cbbe..b2c52cf2a95a44 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_io__BufferedIOBase_readinto__doc__, "readinto($self, buffer, /)\n" @@ -1099,4 +1098,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=768c3a3a3deabcb4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9e09091995ae02b0 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index b5cdd0802b8748..d7364779200827 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io_BytesIO_readable__doc__, "readable($self, /)\n" @@ -539,4 +538,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=6b1219bda0619e2a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8ab65edc03edbfe0 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 7b63e36b5038db..29f8cf6aa9a85c 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io_FileIO_close__doc__, "close($self, /)\n" @@ -537,4 +536,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=3a3c6ed7e5e78063 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=238dd48819076434 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index fb96ea132f3dc5..9ac80b687925a4 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -2,12 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io__IOBase_seek__doc__, "seek($self, offset, whence=os.SEEK_SET, /)\n" @@ -442,4 +437,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=d96f5bfd72e6eafb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=95e1633805d10294 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index dbe1833a90ed8c..ccdae18d2af24a 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io_StringIO_getvalue__doc__, "getvalue($self, /)\n" @@ -368,4 +367,4 @@ _io_StringIO_seekable(stringio *self, PyObject *Py_UNUSED(ignored)) { return _io_StringIO_seekable_impl(self); } -/*[clinic end generated code: output=09d4056cc8c4aae4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=57e86cd679344ee7 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index ce6efbe20b6bba..bc14327388fb2f 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "detach($self, /)\n" @@ -980,4 +979,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=390af0e65a0d02c0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=175e1723a462a722 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 53f971e37c1258..0683eecdfebb37 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #if defined(HAVE_WINDOWS_CONSOLE_IO) @@ -466,4 +465,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=0cdb16d95f1c7dac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7be51d48ddb7c8c8 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/multiprocessing.c.h b/Modules/_multiprocessing/clinic/multiprocessing.c.h index e8eed0c0c4283f..0ae0039a666061 100644 --- a/Modules/_multiprocessing/clinic/multiprocessing.c.h +++ b/Modules/_multiprocessing/clinic/multiprocessing.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - #if defined(MS_WINDOWS) PyDoc_STRVAR(_multiprocessing_closesocket__doc__, @@ -172,4 +166,4 @@ _multiprocessing_sem_unlink(PyObject *module, PyObject *arg) #ifndef _MULTIPROCESSING_SEND_METHODDEF #define _MULTIPROCESSING_SEND_METHODDEF #endif /* !defined(_MULTIPROCESSING_SEND_METHODDEF) */ -/*[clinic end generated code: output=eb078dff6b6595ff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8b91c020d4353cc5 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 265f46889d531b..09dfa11da23b86 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - #if defined(HAVE_SHM_OPEN) PyDoc_STRVAR(_posixshmem_shm_open__doc__, @@ -166,4 +165,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=c022e2f7371fdece input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2f356903a281d857 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index c5bffc843570d8..a4357376ee1292 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - #if defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) PyDoc_STRVAR(_multiprocessing_SemLock_acquire__doc__, @@ -542,4 +541,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=0fcadfdd8d6944be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e8ea65f8cba8e173 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index 998c8de1e09f16..eceb5a7bf23673 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(pysqlite_connect__doc__, "connect($module, /, database, timeout=5.0, detect_types=0,\n" " isolation_level=\'\', check_same_thread=True,\n" @@ -28,4 +27,4 @@ PyDoc_STRVAR(pysqlite_connect__doc__, #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=8d49736db880f09a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=03bd99542e3aec9d input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 7dda4d77c72ab1..e7c60264722f7b 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(blob_close__doc__, "close($self, /)\n" "--\n" @@ -219,4 +213,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=1783b816ccc3bb75 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8bfd79ab12ac5385 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index f9510f47dd9efa..fcc657799dfaf0 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - static int pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, double timeout, int detect_types, @@ -1822,4 +1821,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=f06b254bc5c2bcaf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=166bf41ad5ca1655 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index 1a21a1c41dafd8..3cb637bde4ff6c 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - static int pysqlite_cursor_init_impl(pysqlite_Cursor *self, pysqlite_Connection *connection); @@ -313,4 +312,4 @@ pysqlite_cursor_close(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl(self); } -/*[clinic end generated code: output=b56b3ddb3b6df8c6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0c52a9cf54d00543 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 82e628ababd4a1..49ba7a341ed587 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(pysqlite_complete_statement__doc__, "complete_statement($module, /, statement)\n" "--\n" @@ -208,4 +207,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c1d450089867b4bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a14893a7c2eead5e input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/row.c.h b/Modules/_sqlite/clinic/row.c.h index 89a48fd52da226..cdf850d01e01a3 100644 --- a/Modules/_sqlite/clinic/row.c.h +++ b/Modules/_sqlite/clinic/row.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - static PyObject * pysqlite_row_new_impl(PyTypeObject *type, pysqlite_Cursor *cursor, PyObject *data); @@ -60,4 +54,4 @@ pysqlite_row_keys(pysqlite_Row *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_row_keys_impl(self); } -/*[clinic end generated code: output=157b31ac3f6af1ba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=972487d535d2e7d5 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index 1846f175b4585a..1b5e1a710af9db 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_sre_getcodesize__doc__, "getcodesize($module, /)\n" @@ -1461,4 +1460,4 @@ _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=b8cf77f05e44d08c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=46d83927cbafa93a input=a9049054013a1b77]*/ diff --git a/Modules/_ssl/clinic/cert.c.h b/Modules/_ssl/clinic/cert.c.h index e2842263bec04d..db43c88e411139 100644 --- a/Modules/_ssl/clinic/cert.c.h +++ b/Modules/_ssl/clinic/cert.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_ssl_Certificate_public_bytes__doc__, "public_bytes($self, /, format=Encoding.PEM)\n" "--\n" @@ -86,4 +85,4 @@ _ssl_Certificate_get_info(PySSLCertificate *self, PyObject *Py_UNUSED(ignored)) { return _ssl_Certificate_get_info_impl(self); } -/*[clinic end generated code: output=72e2bb139c64546c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e438b54fbebd53e input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 61a9103b97f5b4..01881534329c9d 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_testcapi_err_set_raised__doc__, "err_set_raised($module, exception, /)\n" "--\n" @@ -488,4 +487,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=6c4b7ad1cb1e0153 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f273949da28ffb5 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/float.c.h b/Modules/_testcapi/clinic/float.c.h index 84de17a8ee1762..fb0931a0703e11 100644 --- a/Modules/_testcapi/clinic/float.c.h +++ b/Modules/_testcapi/clinic/float.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_testcapi_float_pack__doc__, "float_pack($module, size, d, le, /)\n" "--\n" @@ -85,4 +79,4 @@ _testcapi_float_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=de6879d0f4987d79 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=50146051f1341cce input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index 87bba4cfacfe55..b77cb51810cb65 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_testcapi_test_long_api__doc__, "test_long_api($module, /)\n" "--\n" @@ -171,4 +165,4 @@ PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, #define _TESTCAPI_PYLONG_ASINT_METHODDEF \ {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=1631a18f1193486a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31267ab2dd90aa1d input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/vectorcall.c.h b/Modules/_testcapi/clinic/vectorcall.c.h index 728c0d382565a7..48688e9ade050a 100644 --- a/Modules/_testcapi/clinic/vectorcall.c.h +++ b/Modules/_testcapi/clinic/vectorcall.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_testcapi_pyobject_fastcalldict__doc__, "pyobject_fastcalldict($module, func, func_args, kwargs, /)\n" "--\n" @@ -210,4 +204,4 @@ _testcapi_has_vectorcall_flag(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=beaf6beac3d13c25 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0667266b825ec9ec input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/watchers.c.h b/Modules/_testcapi/clinic/watchers.c.h index ce9c770a37cc5f..fd2ef608334049 100644 --- a/Modules/_testcapi/clinic/watchers.c.h +++ b/Modules/_testcapi/clinic/watchers.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_testcapi_watch_dict__doc__, "watch_dict($module, watcher_id, dict, /)\n" "--\n" @@ -195,4 +189,4 @@ _testcapi_set_func_kwdefaults_via_capi(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=b7564a84c5815b46 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5ad5771d6b29dfb9 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 1b41c231eac5d8..0b73a7059eb584 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "encode($self, /, input, errors=None)\n" "--\n" @@ -690,4 +689,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=5f0e8dacddb0ac76 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ee928e7a85e9d34 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 2adec818c91311..8d3832e1b83d2d 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_abc__reset_registry__doc__, "_reset_registry($module, self, /)\n" "--\n" @@ -165,4 +159,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=c2e69611a495c98d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=babb3ce445fa9b21 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 6a780a80cd0bc4..74dd06461e6e27 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_asyncio_Future___init____doc__, "Future(*, loop=None)\n" "--\n" @@ -1487,4 +1486,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=6b0e283177b07639 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b7658bfab7024f3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 62ce0fc8172367..5553b05acbf521 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_bisect_bisect_right__doc__, "bisect_right($module, /, a, x, lo=0, hi=None, *, key=None)\n" @@ -434,4 +433,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=839fdddeacdc2ecb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=43ece163c3e972df input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 7c4d57b5fec7e1..29f5d3a68e06ae 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_bz2_BZ2Compressor_compress__doc__, "compress($self, data, /)\n" @@ -242,4 +241,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=40706688f92642b4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3dfc8436fa8eaefb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index fc6dcaa7651287..bf9bd4a61f5ce4 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_codecs_register__doc__, "register($module, search_function, /)\n" "--\n" @@ -2818,4 +2817,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=6d9bc20c8dd36134 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3473564544f10403 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index b96502d3cf4b3f..a375b87261ce2d 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -2,12 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_collections__count_elements__doc__, "_count_elements($module, mapping, iterable, /)\n" @@ -76,4 +71,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d83188040656ad7c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b01ddb9fdecc4a2d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_contextvarsmodule.c.h b/Modules/clinic/_contextvarsmodule.c.h index 461d4845635ef0..b1885e41c355d2 100644 --- a/Modules/clinic/_contextvarsmodule.c.h +++ b/Modules/clinic/_contextvarsmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_contextvars_copy_context__doc__, "copy_context($module, /)\n" "--\n" @@ -24,4 +18,4 @@ _contextvars_copy_context(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _contextvars_copy_context_impl(module); } -/*[clinic end generated code: output=1736c27450823e70 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=26e07024451baf52 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index 8900946350a524..86ed0efd79028e 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_csv_list_dialects__doc__, "list_dialects($module, /)\n" "--\n" @@ -206,4 +205,4 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=94374e41eb2806ee input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4704d708f5745e91 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 900058e6d59af0..a8c32c6aa3fc10 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "bottom($self, /)\n" "--\n" @@ -422,4 +416,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=cb76cb1d5040f387 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dd302cb9afc42f40 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 836087581a73ba..ecc3c059d03c90 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_curses_window_addch__doc__, "addch([y, x,] ch, [attr=_curses.A_NORMAL])\n" "Paint the character.\n" @@ -4313,4 +4312,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=cff0e570de65d9b8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=839faafb638935ea input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index c058c13fcc0683..177a37e486e24d 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "fromtimestamp($type, timestamp, /)\n" "--\n" @@ -146,4 +145,4 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=720c99f24a9aa41c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e422c25b1c28f38b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index fbf4ccf27c1f71..a76b8b30eded39 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_dbm_dbm_close__doc__, "close($self, /)\n" "--\n" @@ -222,4 +216,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f1bf74536f201669 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=972d221f9da819d3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 78e456b6a43463..3477257f9d2cf2 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_elementtree_Element_append__doc__, "append($self, subelement, /)\n" @@ -1219,4 +1218,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=0a34620406b95eb0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=20d1869da79a43d7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 9c79e6430413bf..25a3e5b0da60a8 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_functools_cmp_to_key__doc__, "cmp_to_key($module, /, mycmp)\n" "--\n" @@ -101,4 +100,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig { return _functools__lru_cache_wrapper_cache_clear_impl(self); } -/*[clinic end generated code: output=7e7f3bcf9ed61f23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de9cf85e85167f43 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 73e2d7934213ee..e9138bbf0430c5 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_gdbm_gdbm_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -344,4 +338,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=cf48d9f76fdd62b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=84f30c7fff0eadac input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 0ea192eac2d3f1..e41f608bde5011 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(EVP_copy__doc__, "copy($self, /)\n" @@ -1852,4 +1851,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=cfad8d5e904a4917 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=75413752099f2dec input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index 3ee3f51702fa30..8d73b5b48d6a0e 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_heapq_heappush__doc__, "heappush($module, heap, item, /)\n" "--\n" @@ -271,4 +265,4 @@ _heapq__heapify_max(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=29e99a48c57f82bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9a22715a8bf0c91d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_localemodule.c.h b/Modules/clinic/_localemodule.c.h index 18cf5025012e7c..efbb9f709958cd 100644 --- a/Modules/clinic/_localemodule.c.h +++ b/Modules/clinic/_localemodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_locale_setlocale__doc__, "setlocale($module, category, locale=, /)\n" "--\n" @@ -599,4 +593,4 @@ _locale_getencoding(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #define _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #endif /* !defined(_LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF) */ -/*[clinic end generated code: output=119644f17919234d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3abe7fade999eff6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index 5fcc7ae02e3b00..dfc003eb54774c 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_lsprof_Profiler_getstats__doc__, "getstats($self, /)\n" "--\n" @@ -51,4 +45,4 @@ _lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *con } return _lsprof_Profiler_getstats_impl(self, cls); } -/*[clinic end generated code: output=7425d3481349629a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0615a53cce828f06 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index ceb75399157627..528b48384f0bcb 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_lzma_LZMACompressor_compress__doc__, "compress($self, data, /)\n" @@ -339,4 +338,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=f1a001f5f489c372 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eadc9ee7a11a06f5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index f8e6a0a54f940c..4b9a70d2ba9852 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_opcode_stack_effect__doc__, "stack_effect($module, opcode, oparg=None, /, *, jump=None)\n" "--\n" @@ -668,4 +667,4 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=ce89aee80dd825d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d608239a4c7a05a1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_operator.c.h b/Modules/clinic/_operator.c.h index 18b0b920b0d118..06bde243b4e85b 100644 --- a/Modules/clinic/_operator.c.h +++ b/Modules/clinic/_operator.c.h @@ -2,12 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_operator_truth__doc__, "truth($module, a, /)\n" @@ -1493,4 +1488,4 @@ _operator__compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=72bc63a775937245 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9658aca50a9ad991 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 539acc34a05cc1..5df40d62827754 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_pickle_Pickler_clear_memo__doc__, "clear_memo($self, /)\n" "--\n" @@ -1034,4 +1033,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=a0e04b85e7bae626 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=57c209a12264146d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index 3b19a47e716492..83048a385d319c 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(subprocess_fork_exec__doc__, "fork_exec($module, args, executable_list, close_fds, pass_fds, cwd,\n" " env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" @@ -159,4 +153,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=be0e9b5d8c0217fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a83b11467169b97b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 94fb59a5b17a49..8d4df14ae3b009 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(simplequeue_new__doc__, "SimpleQueue()\n" "--\n" @@ -331,4 +330,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=9a72a8d1b5767f6a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c326e4c1f2a1ad7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 45f06b60ed4448..008b531d832c5a 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_random_Random_random__doc__, "random($self, /)\n" "--\n" @@ -115,4 +109,4 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=3feabc783b317d0d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e7e05d756a7e1c7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 6ab8e3ee436336..5f8ae29180a4fa 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, "do_handshake($self, /)\n" @@ -1543,4 +1542,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=6e2eb86330f3f9b8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a47d575abe0aceb6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_statisticsmodule.c.h b/Modules/clinic/_statisticsmodule.c.h index 4dedadd2939ad6..03543e41af7f5a 100644 --- a/Modules/clinic/_statisticsmodule.c.h +++ b/Modules/clinic/_statisticsmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_statistics__normal_dist_inv_cdf__doc__, "_normal_dist_inv_cdf($module, p, mu, sigma, /)\n" "--\n" @@ -71,4 +65,4 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi exit: return return_value; } -/*[clinic end generated code: output=6899dc752cc6b457 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b807a8243e7801e6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index a823364447818b..8c468e6e7205e6 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(Struct__doc__, "Struct(format)\n" @@ -452,4 +451,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=9b48aeaa86898ec5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c1bc384ff87df1f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 94c4b5a1221eec..ba3aeca49e26eb 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -3,12 +3,11 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head #endif - -#include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_runtime.h" // _Py_ID() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -3071,4 +3070,4 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=a652e6b1787d3346 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c2a69a08ffdfc466 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 7b9d8971efb46b..36ff55bc002939 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -3,12 +3,11 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head #endif - -#include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_runtime.h" // _Py_ID() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -2393,4 +2392,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=464aeba97e482f5c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=689b1e2d0872e413 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 72a5f166099659..28761387023508 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_testinternalcapi_compiler_cleandoc__doc__, "compiler_cleandoc($module, /, doc)\n" "--\n" @@ -265,4 +264,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=1ce6e2257e95a853 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cba1c94ff4015b82 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index f7b99b1c52f0a9..a96d20bf33ea03 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "get_defining_module($self, /)\n" "--\n" @@ -162,4 +161,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=6f8db3983c5312a0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db1fdd15244ee59c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index e588ea6989c672..fcfc406238808b 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_tkinter_tkapp_eval__doc__, "eval($self, script, /)\n" "--\n" @@ -865,4 +859,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=2cdb2fd0ce66ebe5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bcd9cdc8f6bdcfae input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tracemalloc.c.h b/Modules/clinic/_tracemalloc.c.h index aa4fe84939aaa2..c07ad797d6295b 100644 --- a/Modules/clinic/_tracemalloc.c.h +++ b/Modules/clinic/_tracemalloc.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_tracemalloc_is_tracing__doc__, "is_tracing($module, /)\n" "--\n" @@ -218,4 +212,4 @@ _tracemalloc_reset_peak(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _tracemalloc_reset_peak_impl(module); } -/*[clinic end generated code: output=98a42e95af7eaf09 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ad7d1fae89f2bdaa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_typingmodule.c.h b/Modules/clinic/_typingmodule.c.h index f980aa0d0844da..ea415e67153ed8 100644 --- a/Modules/clinic/_typingmodule.c.h +++ b/Modules/clinic/_typingmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_typing__idfunc__doc__, "_idfunc($module, x, /)\n" "--\n" @@ -15,4 +9,4 @@ PyDoc_STRVAR(_typing__idfunc__doc__, #define _TYPING__IDFUNC_METHODDEF \ {"_idfunc", (PyCFunction)_typing__idfunc, METH_O, _typing__idfunc__doc__}, -/*[clinic end generated code: output=97457fda45072c7d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e7ea2a3cb7ab301a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_weakref.c.h b/Modules/clinic/_weakref.c.h index 48feb042cac039..541cba75e6813c 100644 --- a/Modules/clinic/_weakref.c.h +++ b/Modules/clinic/_weakref.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_weakref_getweakrefcount__doc__, "getweakrefcount($module, object, /)\n" "--\n" @@ -116,4 +110,4 @@ _weakref_proxy(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=28265e89d583273d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f4be6b8177fbceb8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index c648e68be448c6..7fef127bef8606 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_long.h" // _PyLong_Size_t_Converter() +#include "pycore_long.h" // _PyLong_Size_t_Converter() PyDoc_STRVAR(_winapi_Overlapped_GetOverlappedResult__doc__, "GetOverlappedResult($self, wait, /)\n" @@ -1479,4 +1478,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=c7e08927e163ef13 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6df38b5eb93f2e5a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index ae62865e0f67df..6691d39deb7c24 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "from_file($type, file_obj, /, key=None)\n" "--\n" @@ -372,4 +371,4 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con exit: return return_value; } -/*[clinic end generated code: output=54051388dfc408af input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a5384f79d49a593b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 8bdfcc62bef085..2a874b8c830609 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -2,12 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(array_array___copy____doc__, "__copy__($self, /)\n" @@ -675,4 +670,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=ab24c7a40a41c2e1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8595b1906b5a6552 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 8af49b4d87afc2..d80decf145bfdc 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(binascii_a2b_uu__doc__, "a2b_uu($module, data, /)\n" "--\n" @@ -795,4 +794,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=bf950ef45a10928d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=acc9419209dfd568 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index 941448e76e80de..a665ac3c1c4994 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(cmath_acos__doc__, "acos($module, z, /)\n" "--\n" @@ -982,4 +981,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=87f609786ef270cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96a5c4ae198dd5bf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index c15f345ee6ffe8..83e882bbbaf184 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,12 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" @@ -250,4 +245,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5f096e8731fa38be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4d4fac195494faec input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 290a8aaa2728e5..40448404877e3e 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -425,4 +424,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=fe89ce0b90a1067d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=63093e7724b94a37 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/grpmodule.c.h b/Modules/clinic/grpmodule.c.h index 38de3ce68ceb7f..e6c9cccfce4a76 100644 --- a/Modules/clinic/grpmodule.c.h +++ b/Modules/clinic/grpmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(grp_getgrgid__doc__, "getgrgid($module, /, id)\n" "--\n" @@ -146,4 +145,4 @@ grp_getgrall(PyObject *module, PyObject *Py_UNUSED(ignored)) { return grp_getgrall_impl(module); } -/*[clinic end generated code: output=e685227ed5d9be9f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d4bdad9b26fb8558 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 6555bd495b200e..74cd3ddc3c9a86 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(batched_new__doc__, "batched(iterable, n)\n" @@ -914,4 +913,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=20999801c7349d2c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=95bb863274717bd9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index c16c1b083985f2..bfce397be77c8b 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" "--\n" @@ -950,4 +949,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=91a0357265a2a553 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=da62cea7eb3dc8bc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index b4602104f18042..6087a0f9c25c1a 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(MD5Type_copy__doc__, "copy($self, /)\n" "--\n" @@ -148,4 +147,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=b4924c9905cc9f34 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=943d42b9d17d9a5b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 100df2062d8efb..9a71e25aa557a4 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(_overlapped_CreateIoCompletionPort__doc__, "CreateIoCompletionPort($module, handle, port, key, concurrency, /)\n" @@ -1263,4 +1262,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=994ad727b827ff87 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31bcc780209593a2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index e80bbff6b5156d..854fd6cf5ec353 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3,13 +3,12 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_long.h" // _PyLong_UnsignedInt_Converter() +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_long.h" // _PyLong_UnsignedInt_Converter() PyDoc_STRVAR(os_stat__doc__, "stat($module, /, path, *, dir_fd=None, follow_symlinks=True)\n" @@ -11993,4 +11992,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=1b34619e5f65adc2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=30c63cb556431cea input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 748c873b1c8a45..0e8aa5e6a9f470 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(pwd_getpwuid__doc__, "getpwuid($module, uidobj, /)\n" "--\n" @@ -77,4 +71,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=1edf1e26cd2762db input=a9049054013a1b77]*/ +/*[clinic end generated code: output=211c7a2516899b91 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 97247781eaa072..21b09110d00853 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "Parse($self, data, isfinal=False, /)\n" "--\n" @@ -498,4 +497,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=6b30852bfc778208 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ac398d94833e802d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index 0bd6e36851e45d..e5a784a5f79e5a 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(readline_parse_and_bind__doc__, "parse_and_bind($module, string, /)\n" "--\n" @@ -688,4 +682,4 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=2c9c5709b3e00c8b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=790ed2ba01babb60 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index d769a7f612357d..0ac272af25d154 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - #if defined(HAVE_GETRUSAGE) PyDoc_STRVAR(resource_getrusage__doc__, @@ -178,4 +172,4 @@ resource_getpagesize(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=916ecf0eb1927c37 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2376b9a3f03777aa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 69c0f06789e714..282196b8f88973 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -3,12 +3,11 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() PyDoc_STRVAR(select_select__doc__, "select($module, rlist, wlist, xlist, timeout=None, /)\n" @@ -1311,4 +1310,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=a215af2157f038c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=35cd4aa7b6802838 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index ad15ddaadfc86c..fa4ec8bb51e787 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(SHA1Type_copy__doc__, "copy($self, /)\n" "--\n" @@ -148,4 +147,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=4d1293ca3472acdb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7efbe42154a7a7f8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index 8f855ca345e47a..9fb8bf7de876fa 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(SHA256Type_copy__doc__, "copy($self, /)\n" "--\n" @@ -437,4 +436,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=f81dacb48f3fee72 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1242ccc50bcefe98 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 80fe3ed32c80dc..fd65462d62c195 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" @@ -194,4 +193,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=5188d9ae4af48c6d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0888aed37ef39984 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index f0ae699a3f1e18..5e8957a4be780d 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(signal_default_int_handler__doc__, "default_int_handler($module, signalnum, frame, /)\n" "--\n" @@ -705,4 +699,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=9903be3eba6cbc1c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef4c2ad1a2443063 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 5e2f581c3e0600..c62050160f1117 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - static int sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, PyObject *fdobj); @@ -91,4 +90,4 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c44e3ac23999ac60 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b68ae94d6cbeeb1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 04fdb9f2d9b776..2cd08f81782007 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_symtable_symtable__doc__, "symtable($module, source, filename, startstr, /)\n" "--\n" @@ -54,4 +48,4 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=07716ddbd6c7efe1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3f7ccf535d750238 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index 3e0792b88060c0..f0c6f2a871d042 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(syslog_openlog__doc__, "openlog($module, /, ident=, logoption=0,\n" " facility=LOG_USER)\n" @@ -251,4 +250,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=b8124c0977ed6177 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=68942dad7fb3872e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 44d4107c378e53..78828e57d73077 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -2,12 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() PyDoc_STRVAR(termios_tcgetattr__doc__, "tcgetattr($module, fd, /)\n" @@ -293,4 +288,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=434df4394b596e92 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=daecc8ebc8fe29f5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h index ec30f9017e3702..74362c6a6857b7 100644 --- a/Modules/clinic/unicodedata.c.h +++ b/Modules/clinic/unicodedata.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(unicodedata_UCD_decimal__doc__, "decimal($self, chr, default=, /)\n" "--\n" @@ -523,4 +517,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=43e551ecaa985a40 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ca10ded25747d182 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 1debe2ae98959d..895215307974f7 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(zlib_compress__doc__, "compress($module, data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS)\n" @@ -1130,4 +1129,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=d574a79aa47c9969 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d90c72ba2dd04c5 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index f091fc7afbd693..7039db09721998 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() static int bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, @@ -1285,4 +1284,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=d388e9027b333f00 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=94b9b5f492b5fed6 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 66db76dbbb4faf..e2f9ba6c1e55bd 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(bytes___bytes____doc__, "__bytes__($self, /)\n" @@ -1061,4 +1060,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=9da56b6c04914e18 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a9f5c28cbfe7592 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/classobject.c.h b/Objects/clinic/classobject.c.h index a7bac63052bc40..48cfd6c7b78ca3 100644 --- a/Objects/clinic/classobject.c.h +++ b/Objects/clinic/classobject.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(method___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -86,4 +80,4 @@ instancemethod_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=2a5e7fa5947a86cb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a0d17bad3b0734d9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 65801bd35156a6..b20b066f494901 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(code_new__doc__, "code(argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize,\n" " flags, codestring, constants, names, varnames, filename, name,\n" @@ -464,4 +463,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=0446968a1fbd13b2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b9ccfbfabe1a5f46 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index bb2b3815415f85..788f30d54450f5 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(complex_conjugate__doc__, "conjugate($self, /)\n" "--\n" @@ -157,4 +156,4 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d438b7ed87f8459e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=002c74f8a33b6697 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/descrobject.c.h b/Objects/clinic/descrobject.c.h index 75706437df83f9..4f18fd77a2c7a0 100644 --- a/Objects/clinic/descrobject.c.h +++ b/Objects/clinic/descrobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping); @@ -167,4 +166,4 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8dc1ddfcf764ac8e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a97dc44d12f9f9b6 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index bc2452330e4e2f..eda86c31fcc578 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(dict_fromkeys__doc__, "fromkeys($type, iterable, value=None, /)\n" "--\n" @@ -197,4 +191,4 @@ dict___reversed__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict___reversed___impl(self); } -/*[clinic end generated code: output=c0064abbea6091c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=582766ac0154c8bf input=a9049054013a1b77]*/ diff --git a/Objects/clinic/enumobject.c.h b/Objects/clinic/enumobject.c.h index adf78efd0d66f4..97f20c63cd4adf 100644 --- a/Objects/clinic/enumobject.c.h +++ b/Objects/clinic/enumobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(enum_new__doc__, "enumerate(iterable, start=0)\n" "--\n" @@ -107,4 +106,4 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=aba0ddbeab1601e3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=661b29708f501d19 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 3d9cd3a683ff5f..c40eff85cf3552 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(float_is_integer__doc__, "is_integer($self, /)\n" "--\n" @@ -322,4 +316,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=355c3f5102034a41 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eb093cc601cc5426 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/funcobject.c.h b/Objects/clinic/funcobject.c.h index c3a3a8edc39278..1be45674286905 100644 --- a/Objects/clinic/funcobject.c.h +++ b/Objects/clinic/funcobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(func_new__doc__, "function(code, globals, name=None, argdefs=None, closure=None)\n" "--\n" @@ -104,4 +103,4 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=777cead7b1f6fad3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b2d676ff51c992d0 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index a41ed3e4b040d9..b6f0c5c0c42dd4 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(list_insert__doc__, "insert($self, index, object, /)\n" @@ -384,4 +383,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=537a17b562c57505 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e2d9f4092498a5ca input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 3ecef1f0fd54d4..9288648fe3b7c2 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() static PyObject * long_new_impl(PyTypeObject *type, PyObject *x, PyObject *obase); @@ -476,4 +475,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=ea9c87ea532dadbe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=009a537ab558763c input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index a6561e2e54df80..74c749e0b6ccc8 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(memoryview__doc__, "memoryview(object)\n" "--\n" @@ -413,4 +412,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=28a632db32b44b63 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ebdadda3b0fcd35 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/moduleobject.c.h b/Objects/clinic/moduleobject.c.h index 29df3888531933..ce21e4728dcb9e 100644 --- a/Objects/clinic/moduleobject.c.h +++ b/Objects/clinic/moduleobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(module___init____doc__, "module(name, doc=None)\n" "--\n" @@ -74,4 +73,4 @@ module___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a5a750cc8190576e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d3d7854d17a033c input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 115a134e3f7f54..643f504535d257 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(OrderedDict_fromkeys__doc__, "fromkeys($type, /, iterable, value=None)\n" "--\n" @@ -332,4 +331,4 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=76d85a9162d62ca8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d7ae9fb552c6108 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/structseq.c.h b/Objects/clinic/structseq.c.h index 40ba18a544f4b3..2571888454cb79 100644 --- a/Objects/clinic/structseq.c.h +++ b/Objects/clinic/structseq.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - static PyObject * structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict); @@ -62,4 +61,4 @@ structseq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=802d5663c7d01024 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2f88fe2a6f5c13a8 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/tupleobject.c.h b/Objects/clinic/tupleobject.c.h index 3de95759a13f21..f21712746a5ad1 100644 --- a/Objects/clinic/tupleobject.c.h +++ b/Objects/clinic/tupleobject.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(tuple_index__doc__, "index($self, value, start=0, stop=sys.maxsize, /)\n" "--\n" @@ -118,4 +112,4 @@ tuple___getnewargs__(PyTupleObject *self, PyObject *Py_UNUSED(ignored)) { return tuple___getnewargs___impl(self); } -/*[clinic end generated code: output=48a9e0834b300ac3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7c5d9d12e0cf6a83 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typeobject.c.h b/Objects/clinic/typeobject.c.h index 1857eab17a793b..eb0b22a943ae8f 100644 --- a/Objects/clinic/typeobject.c.h +++ b/Objects/clinic/typeobject.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(type___instancecheck____doc__, "__instancecheck__($self, instance, /)\n" "--\n" @@ -266,4 +260,4 @@ object___dir__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return object___dir___impl(self); } -/*[clinic end generated code: output=baf5ec2093815a3f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=943f639f264362d9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 5d12be5361cd2f..4762cd63545b60 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(typevar_new__doc__, "typevar(name, *constraints, *, bound=None, covariant=False,\n" " contravariant=False, infer_variance=False)\n" @@ -591,4 +590,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d614edf64f28e346 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db0b327ebbb1488f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 56b64087d6654a..cfee9b8aa1fa8c 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(EncodingMap_size__doc__, "size($self, /)\n" @@ -1505,4 +1504,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a64776a3ea1c970c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d08dfbb814c4393 input=a9049054013a1b77]*/ diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index d1adda4e979c14..e81c5916e6f196 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(stringlib_expandtabs__doc__, "expandtabs($self, /, tabsize=8)\n" @@ -279,4 +278,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=a0338b2d41671b17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=39cd1ee983137188 input=a9049054013a1b77]*/ diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h index b2f3b4ce8b08a2..99cd302ff34698 100644 --- a/PC/clinic/_testconsole.c.h +++ b/PC/clinic/_testconsole.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - #if defined(MS_WINDOWS) PyDoc_STRVAR(_testconsole_write_input__doc__, @@ -140,4 +139,4 @@ _testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF #define _TESTCONSOLE_READ_OUTPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ -/*[clinic end generated code: output=208c72e2c873555b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f59fe72cd4e73704 input=a9049054013a1b77]*/ diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index bfcad41750b319..3ece5e6823b462 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_wmi_exec_query__doc__, "exec_query($module, /, query)\n" "--\n" @@ -69,4 +68,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=923d09bee1d15c5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=53821e597fc2aca4 input=a9049054013a1b77]*/ diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index af6710af9657e4..e2959a0fda1c52 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(msvcrt_heapmin__doc__, "heapmin($module, /)\n" "--\n" @@ -701,4 +695,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=f70de1b6d0e700cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=97e00f191821d4c0 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 6e23929c185caf..7507c1151a9840 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() #if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) @@ -1762,4 +1761,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=2431b1b06b148722 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d7ae41899af53d7c input=a9049054013a1b77]*/ diff --git a/PC/clinic/winsound.c.h b/PC/clinic/winsound.c.h index d35261253557da..b8c8e7d04a699f 100644 --- a/PC/clinic/winsound.c.h +++ b/PC/clinic/winsound.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(winsound_PlaySound__doc__, "PlaySound($module, /, sound, flags)\n" "--\n" @@ -206,4 +205,4 @@ winsound_MessageBeep(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=e63a6516d7a55cb8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=21584101f656198f input=a9049054013a1b77]*/ diff --git a/Python/clinic/Python-tokenize.c.h b/Python/clinic/Python-tokenize.c.h index 28f5075826e36f..a8453e25807b0d 100644 --- a/Python/clinic/Python-tokenize.c.h +++ b/Python/clinic/Python-tokenize.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - static PyObject * tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, int extra_tokens, const char *encoding); @@ -80,4 +79,4 @@ tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=48be65a2808bdfa6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=406b5a433a59069c input=a9049054013a1b77]*/ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index 1a01416e7363c7..cf3c73b1cf0e6e 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_abstract.h" // _PyNumber_Index() PyDoc_STRVAR(warnings_warn__doc__, "warn($module, /, message, category=None, stacklevel=1, source=None, *,\n" @@ -244,4 +243,4 @@ warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) { return warnings_filters_mutated_impl(module); } -/*[clinic end generated code: output=c8a6dc1403fba1d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c396a721ef75739 input=a9049054013a1b77]*/ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index b0d6929aa50dc8..b4dcc6bf1d2978 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(builtin___import____doc__, "__import__($module, /, name, globals=None, locals=None, fromlist=(),\n" " level=0)\n" @@ -1212,4 +1211,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=bb2da8ccae4190e9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=607c62f2341ebfc0 input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index 27c375717bff96..292d3f7f4ff49c 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(_contextvars_Context_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -183,4 +177,4 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__, #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF \ {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__}, -/*[clinic end generated code: output=0c94d4b919500438 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2436b16a92452869 input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index fd54033b7e7ac8..890c3026fc9996 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(_imp_lock_held__doc__, "lock_held($module, /)\n" "--\n" @@ -627,4 +626,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=d97b56ef622cb28a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=058f6aa1c9f4ebe4 input=a9049054013a1b77]*/ diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index c97fb977ad3457..1b1b6d048b0dd0 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(monitoring_use_tool_id__doc__, "use_tool_id($module, tool_id, name, /)\n" "--\n" @@ -308,4 +302,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=8baabc8341df3f0e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=46f449b18195f976 input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index 102e637ec18330..16e48f77c87fdb 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - PyDoc_STRVAR(marshal_dump__doc__, "dump($module, value, file, version=version, /)\n" "--\n" @@ -161,4 +155,4 @@ marshal_loads(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=0f4ecfd941293f67 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=23091e077319f596 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 1f751e48d3e0fe..a4b39873735fd9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(sys_addaudithook__doc__, "addaudithook($module, /, hook)\n" "--\n" @@ -1412,4 +1411,4 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=6de02cd7d925d1de input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6619682ea70e7375 input=a9049054013a1b77]*/ diff --git a/Python/clinic/traceback.c.h b/Python/clinic/traceback.c.h index 1a0eaf9e91b3c1..c5687e30748782 100644 --- a/Python/clinic/traceback.c.h +++ b/Python/clinic/traceback.c.h @@ -3,11 +3,10 @@ preserve [clinic start generated code]*/ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() #endif - PyDoc_STRVAR(tb_new__doc__, "TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno)\n" "--\n" @@ -78,4 +77,4 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=6303f910c04227a4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=57e27eb68d04d842 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ec5bf302416ba8..b744ca7e5c8145 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -77,6 +77,11 @@ "noptargs", "return_value", } + +# '#include "header.h" // reason': column of '//' comment +INCLUDE_COMMENT_COLUMN = 35 + +# match '#define Py_LIMITED_API' LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') @@ -675,7 +680,9 @@ def normalize_snippet( def declare_parser( f: Function, *, - hasformat: bool = False + hasformat: bool = False, + clinic: Clinic, + limited_capi: bool, ) -> str: """ Generates the code template for a static local PyArg_Parser variable, @@ -694,7 +701,11 @@ def declare_parser( p for p in f.parameters.values() if not p.is_positional_only() and not p.is_vararg() ]) - if num_keywords == 0: + if limited_capi: + declarations = """ + #define KWTUPLE NULL + """ + elif num_keywords == 0: declarations = """ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) @@ -723,6 +734,10 @@ def declare_parser( #endif // !Py_BUILD_CORE """ % num_keywords + condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' + clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition) + clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition) + declarations += """ static const char * const _keywords[] = {{{keywords_c} NULL}}; static _PyArg_Parser _parser = {{ @@ -976,6 +991,7 @@ def deprecate_keyword_use( argname_fmt: str | None, *, limited_capi: bool, + clinic: Clinic, ) -> str: assert len(params) > 0 last_param = next(reversed(params.values())) @@ -990,9 +1006,11 @@ def deprecate_keyword_use( elif func.kind.new_or_init or limited_capi: conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") containscheck = "PyDict_Contains" + clinic.add_include('pycore_runtime.h', '_Py_ID()') else: conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") containscheck = "PySequence_Contains" + clinic.add_include('pycore_runtime.h', '_Py_ID()') else: conditions = [f"nargs < {i+1}"] condition = ") || (".join(conditions) @@ -1081,8 +1099,10 @@ def output_templates( # Copy includes from parameters to Clinic for converter in converters: - if converter.include: - clinic.add_include(*converter.include) + include = converter.include + if include: + clinic.add_include(include.filename, include.reason, + condition=include.condition) has_option_groups = parameters and (parameters[0].group or parameters[-1].group) default_return_converter = f.return_converter.type == 'PyObject *' @@ -1415,14 +1435,16 @@ def parser_body( declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" if deprecated_keywords: code = self.deprecate_keyword_use(f, deprecated_keywords, None, - limited_capi=limited_capi) + limited_capi=limited_capi, + clinic=clinic) parser_code.append(code) elif not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS argname_fmt = 'args[%d]' - declarations = declare_parser(f) + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) declarations += "\nPyObject *argsbuf[%s];" % len(converters) if has_optional_kw: declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) @@ -1437,7 +1459,8 @@ def parser_body( flags = "METH_VARARGS|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_KEYWORD argname_fmt = 'fastargs[%d]' - declarations = declare_parser(f) + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) declarations += "\nPyObject *argsbuf[%s];" % len(converters) declarations += "\nPyObject * const *fastargs;" declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" @@ -1457,7 +1480,8 @@ def parser_body( if not limited_capi: if deprecated_keywords: code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, - limited_capi=limited_capi) + limited_capi=limited_capi, + clinic=clinic) parser_code.append(code) add_label: str | None = None @@ -1522,7 +1546,9 @@ def parser_body( if add_label: parser_code.append("%s:" % add_label) else: - declarations = declare_parser(f, hasformat=True) + declarations = declare_parser(f, clinic=clinic, + hasformat=True, + limited_capi=clinic.limited_capi) if not new_or_init: parser_code = [normalize_snippet(""" if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} @@ -1541,7 +1567,8 @@ def parser_body( declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" if deprecated_keywords: code = self.deprecate_keyword_use(f, deprecated_keywords, None, - limited_capi=limited_capi) + limited_capi=limited_capi, + clinic=clinic) parser_code.append(code) if deprecated_positionals: @@ -2169,6 +2196,26 @@ def is_stop_line(line: str) -> bool: return Block(input_output(), dsl_name, output=output) +@dc.dataclass(slots=True, frozen=True) +class Include: + """ + An include like: #include "pycore_long.h" // _Py_ID() + """ + # Example: "pycore_long.h". + filename: str + + # Example: "_Py_ID()". + reason: str + + # None means unconditional include. + # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". + condition: str | None + + def sort_key(self) -> tuple[str, str]: + # order: '#if' comes before 'NO_CONDITION' + return (self.condition or 'NO_CONDITION', self.filename) + + @dc.dataclass(slots=True) class BlockPrinter: language: Language @@ -2180,7 +2227,7 @@ def print_block( *, core_includes: bool = False, limited_capi: bool, - header_includes: dict[str, str], + header_includes: dict[str, Include], ) -> None: input = block.input output = block.output @@ -2209,21 +2256,32 @@ def print_block( write("\n") output = '' - if core_includes: - if not limited_capi: - output += textwrap.dedent(""" - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # include "pycore_gc.h" // PyGC_Head - # include "pycore_runtime.h" // _Py_ID() - #endif - - """) - + if core_includes and header_includes: # Emit optional "#include" directives for C headers - for include, reason in sorted(header_includes.items()): - line = f'#include "{include}"'.ljust(35) + f'// {reason}\n' + output += '\n' + + current_condition: str | None = None + includes = sorted(header_includes.values(), key=Include.sort_key) + for include in includes: + if include.condition != current_condition: + if current_condition: + output += '#endif\n' + current_condition = include.condition + if include.condition: + output += f'{include.condition}\n' + + if current_condition: + line = f'# include "{include.filename}"' + else: + line = f'#include "{include.filename}"' + if include.reason: + comment = f'// {include.reason}\n' + line = line.ljust(INCLUDE_COMMENT_COLUMN - 1) + comment output += line + if current_condition: + output += '#endif\n' + input = ''.join(block.input) output += ''.join(block.output) if output: @@ -2434,9 +2492,8 @@ def __init__( self.modules: ModuleDict = {} self.classes: ClassDict = {} self.functions: list[Function] = [] - # dict: include name => reason - # Example: 'pycore_long.h' => '_PyLong_UnsignedShort_Converter()' - self.includes: dict[str, str] = {} + # dict: include name => Include instance + self.includes: dict[str, Include] = {} self.line_prefix = self.line_suffix = '' @@ -2495,11 +2552,23 @@ def __init__( global clinic clinic = self - def add_include(self, name: str, reason: str) -> None: - if name in self.includes: - # Mention a single reason is enough, no need to list all of them - return - self.includes[name] = reason + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + try: + existing = self.includes[name] + except KeyError: + pass + else: + if existing.condition and not condition: + # If the previous include has a condition and the new one is + # unconditional, override the include. + pass + else: + # Already included, do nothing. Only mention a single reason, + # no need to list all of them. + return + + self.includes[name] = Include(name, reason, condition) def add_destination( self, @@ -3129,10 +3198,7 @@ class CConverter(metaclass=CConverterAutoRegister): # Only set by self_converter. signature_name: str | None = None - # Optional (name, reason) include which generate a line like: - # "#include "name" // reason" - include: tuple[str, str] | None = None - + include: Include | None = None broken_limited_capi: bool = False # keep in sync with self_converter.__init__! @@ -3439,10 +3505,11 @@ def parser_name(self) -> str: else: return self.name - def add_include(self, name: str, reason: str) -> None: + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: if self.include is not None: raise ValueError("a converter only supports a single include") - self.include = (name, reason) + self.include = Include(name, reason, condition) type_checks = { '&PyLong_Type': ('PyLong_Check', 'int'), From c1e2f3b2f70b8a72ea7e1bf792addf62a94ae65d Mon Sep 17 00:00:00 2001 From: Alex Povel Date: Fri, 1 Sep 2023 00:17:32 +0200 Subject: [PATCH 433/598] `ast` docs: Fix incorrect link on `keyword` (#108728) In two places, Sphinx was erroneously adding links to the `keyword` module instead of the `ast.keyword` class --- Doc/library/ast.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 2237a07eb9d8aa..8f5148feb809a3 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -585,7 +585,7 @@ Expressions :class:`Name` or :class:`Attribute` object. Of the arguments: * ``args`` holds a list of the arguments passed by position. - * ``keywords`` holds a list of :class:`keyword` objects representing + * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. When creating a ``Call`` node, ``args`` and ``keywords`` are required, but @@ -2024,7 +2024,7 @@ Function and class definitions * ``name`` is a raw string for the class name * ``bases`` is a list of nodes for explicitly specified base classes. - * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. + * ``keywords`` is a list of :class:`.keyword` nodes, principally for 'metaclass'. Other keywords will be passed to the metaclass, as per `PEP-3115 `_. * ``body`` is a list of nodes representing the code within the class From 3edcf743e88b4ac4431d4b0f3a66048628cf5c70 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 09:13:07 +0200 Subject: [PATCH 434/598] gh-106320: Remove private _PyLong_Sign() (#108743) Move the private _PyLong_Sign() and _PyLong_NumBits() functions to the internal C API (pycore_long.h). Modules/_testcapi/long.c now uses the internal C API. --- Include/cpython/longobject.h | 15 --------------- Include/internal/pycore_long.h | 17 +++++++++++++++++ Modules/_io/_iomodule.c | 1 + Modules/_testcapi/long.c | 5 +++++ 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 32c40b083b7ab6..57834173490c99 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -4,21 +4,6 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); -/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. - v must not be NULL, and must be a normalized long. - There are no error cases. -*/ -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); - -/* _PyLong_NumBits. Return the number of bits needed to represent the - absolute value of a long. For example, this returns 1 for 1 and -1, 2 - for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. - v must not be NULL, and must be a normalized long. - (size_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. -*/ -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index c411ac654387c3..3c253ed7ff556b 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -58,6 +58,23 @@ PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( Py_ssize_t digit_count, digit *digits); +// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. +// v must not be NULL, and must be a normalized long. +// There are no error cases. +// +// Export for '_pickle' shared extension. +PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + +// _PyLong_NumBits. Return the number of bits needed to represent the +// absolute value of a long. For example, this returns 1 for 1 and -1, 2 +// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. +// v must not be NULL, and must be a normalized long. +// (size_t)-1 is returned and OverflowError set if the true result doesn't +// fit in a size_t. +// +// Export for 'math' shared extension. +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + /* runtime lifecycle */ diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index cc844527da5838..173f5b55e5f732 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_long.h" // _PyLong_Sign() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 6b74e0ab8e0d1c..c1d2d42a2c434e 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -1,5 +1,10 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "parts.h" #include "clinic/long.c.h" +#include "pycore_long.h" // _PyLong_Sign() /*[clinic input] module _testcapi From 044b8b3b6a65e6651b161e3badfa5d57c666db19 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 1 Sep 2023 11:18:15 +0300 Subject: [PATCH 435/598] gh-107805: Fix signatures of module-level generated functions in `turtle` (#107807) Co-authored-by: Alex Waygood --- Lib/test/test_turtle.py | 20 +++++++++ Lib/turtle.py | 43 +++++++++++-------- ...-08-09-13-49-37.gh-issue-107805.ezem0k.rst | 1 + 3 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 3f9f129a3dd200..14121a590a5026 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -461,5 +461,25 @@ def test_teleport(self): self.assertTrue(tpen.isdown()) +class TestModuleLevel(unittest.TestCase): + def test_all_signatures(self): + import inspect + + known_signatures = { + 'teleport': + '(x=None, y=None, *, fill_gap: bool = False) -> None', + 'undo': '()', + 'goto': '(x, y=None)', + 'bgcolor': '(*args)', + 'pen': '(pen=None, **pendict)', + } + + for name in known_signatures: + with self.subTest(name=name): + obj = getattr(turtle, name) + sig = inspect.signature(obj) + self.assertEqual(str(sig), known_signatures[name]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/turtle.py b/Lib/turtle.py index e542bc956897e9..6119b7a447e7bf 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -3920,28 +3920,33 @@ def getmethparlist(ob): function definition and the second is suitable for use in function call. The "self" parameter is not included. """ - defText = callText = "" + orig_sig = inspect.signature(ob) # bit of a hack for methods - turn it into a function # but we drop the "self" param. # Try and build one for Python defined functions - args, varargs, varkw = inspect.getargs(ob.__code__) - items2 = args[1:] - realArgs = args[1:] - defaults = ob.__defaults__ or [] - defaults = ["=%r" % (value,) for value in defaults] - defaults = [""] * (len(realArgs)-len(defaults)) + defaults - items1 = [arg + dflt for arg, dflt in zip(realArgs, defaults)] - if varargs is not None: - items1.append("*" + varargs) - items2.append("*" + varargs) - if varkw is not None: - items1.append("**" + varkw) - items2.append("**" + varkw) - defText = ", ".join(items1) - defText = "(%s)" % defText - callText = ", ".join(items2) - callText = "(%s)" % callText - return defText, callText + func_sig = orig_sig.replace( + parameters=list(orig_sig.parameters.values())[1:], + ) + + call_args = [] + for param in func_sig.parameters.values(): + match param.kind: + case ( + inspect.Parameter.POSITIONAL_ONLY + | inspect.Parameter.POSITIONAL_OR_KEYWORD + ): + call_args.append(param.name) + case inspect.Parameter.VAR_POSITIONAL: + call_args.append(f'*{param.name}') + case inspect.Parameter.KEYWORD_ONLY: + call_args.append(f'{param.name}={param.name}') + case inspect.Parameter.VAR_KEYWORD: + call_args.append(f'**{param.name}') + case _: + raise RuntimeError('Unsupported parameter kind', param.kind) + call_text = f'({', '.join(call_args)})' + + return str(func_sig), call_text def _turtle_docrevise(docstr): """To reduce docstrings from RawTurtle class for functions diff --git a/Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst b/Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst new file mode 100644 index 00000000000000..263df68f8e5c80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst @@ -0,0 +1 @@ +Fix signatures of module-level generated functions in :mod:`turtle`. From 844f4c2e12a7c637d1de93dbbb0718be06553510 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:16:09 +0100 Subject: [PATCH 436/598] gh-108727: Fix segfault due to missing tp_dealloc definition for CounterOptimizer_Type (GH-108734) --- Lib/test/test_capi/test_misc.py | 7 +++++++ .../2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst | 2 ++ Python/optimizer.c | 1 + 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4148f15b2aa662..004ce397696556 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2284,6 +2284,13 @@ def clear_executors(func): class TestOptimizerAPI(unittest.TestCase): + def test_get_counter_optimizer_dealloc(self): + # See gh-108727 + def f(): + _testinternalcapi.get_counter_optimizer() + + f() + def test_get_set_optimizer(self): old = _testinternalcapi.get_optimizer() opt = _testinternalcapi.get_counter_optimizer() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst new file mode 100644 index 00000000000000..34959ae3bb99ac --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst @@ -0,0 +1,2 @@ +Define ``tp_dealloc`` for ``CounterOptimizer_Type``. This fixes a segfault +on deallocation. diff --git a/Python/optimizer.c b/Python/optimizer.c index c311a03fa59a2f..7472f52c50b766 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -289,6 +289,7 @@ static PyTypeObject CounterOptimizer_Type = { .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_methods = counter_methods, + .tp_dealloc = (destructor)PyObject_Del, }; PyObject * From b936cf4fe084474970ea43dba24386f1b08e75ce Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 12:43:30 +0200 Subject: [PATCH 437/598] gh-108634: PyInterpreterState_New() no longer calls Py_FatalError() (#108748) pycore_create_interpreter() now returns a status, rather than calling Py_FatalError(). * PyInterpreterState_New() now calls Py_ExitStatusException() instead of calling Py_FatalError() directly. * Replace Py_FatalError() with PyStatus in init_interpreter() and _PyObject_InitState(). * _PyErr_SetFromPyStatus() now raises RuntimeError, instead of ValueError. It can now call PyErr_NoMemory(), raise MemoryError, if it detects _PyStatus_NO_MEMORY() error message. --- Include/internal/pycore_initconfig.h | 7 +- Include/internal/pycore_interp.h | 4 ++ Include/internal/pycore_object.h | 2 +- Objects/object.c | 5 +- Python/initconfig.c | 27 ++++++-- Python/pylifecycle.c | 8 ++- Python/pystate.c | 97 ++++++++++++++++++---------- 7 files changed, 99 insertions(+), 51 deletions(-) diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 6439101ff390ae..c86988234f6a05 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -22,7 +22,7 @@ struct pyruntimestate; #endif #define _PyStatus_OK() \ - (PyStatus){._type = _PyStatus_TYPE_OK,} + (PyStatus){._type = _PyStatus_TYPE_OK} /* other fields are set to 0 */ #define _PyStatus_ERR(ERR_MSG) \ (PyStatus){ \ @@ -30,7 +30,8 @@ struct pyruntimestate; .func = _PyStatus_GET_FUNC(), \ .err_msg = (ERR_MSG)} /* other fields are set to 0 */ -#define _PyStatus_NO_MEMORY() _PyStatus_ERR("memory allocation failed") +#define _PyStatus_NO_MEMORY_ERRMSG "memory allocation failed" +#define _PyStatus_NO_MEMORY() _PyStatus_ERR(_PyStatus_NO_MEMORY_ERRMSG) #define _PyStatus_EXIT(EXITCODE) \ (PyStatus){ \ ._type = _PyStatus_TYPE_EXIT, \ @@ -45,7 +46,7 @@ struct pyruntimestate; do { (err).func = _PyStatus_GET_FUNC(); } while (0) // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject *) _PyErr_SetFromPyStatus(PyStatus status); +PyAPI_FUNC(void) _PyErr_SetFromPyStatus(PyStatus status); /* --- PyWideStringList ------------------------------------------------ */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 9a75b002e5bc1e..f171c546efd53c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -311,6 +311,10 @@ might not be allowed in the current interpreter (i.e. os.fork() would fail). extern int _PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature); +PyAPI_FUNC(PyStatus) _PyInterpreterState_New( + PyThreadState *tstate, + PyInterpreterState **pinterp); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index d842816e673553..daa06ebfbf91a4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -182,7 +182,7 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); -extern void _PyObject_InitState(PyInterpreterState *interp); +extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); diff --git a/Objects/object.c b/Objects/object.c index a4d7111a686bda..7aeda50e9b2757 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2034,7 +2034,7 @@ PyObject _Py_NotImplementedStruct = { }; -void +PyStatus _PyObject_InitState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS @@ -2048,9 +2048,10 @@ _PyObject_InitState(PyInterpreterState *interp) _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, NULL, NULL, &alloc); if (REFCHAIN(interp) == NULL) { - Py_FatalError("_PyObject_InitState() memory allocation failure"); + return _PyStatus_NO_MEMORY(); } #endif + return _PyStatus_OK(); } void diff --git a/Python/initconfig.c b/Python/initconfig.c index 3281b3c6aea45f..f9c5c64f9c8759 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -335,21 +335,34 @@ int PyStatus_IsExit(PyStatus status) int PyStatus_Exception(PyStatus status) { return _PyStatus_EXCEPTION(status); } -PyObject* +void _PyErr_SetFromPyStatus(PyStatus status) { if (!_PyStatus_IS_ERROR(status)) { PyErr_Format(PyExc_SystemError, - "%s() expects an error PyStatus", - _PyStatus_GET_FUNC()); + "_PyErr_SetFromPyStatus() status is not an error"); + return; } - else if (status.func) { - PyErr_Format(PyExc_ValueError, "%s: %s", status.func, status.err_msg); + + const char *err_msg = status.err_msg; + if (err_msg == NULL || strlen(err_msg) == 0) { + PyErr_Format(PyExc_SystemError, + "_PyErr_SetFromPyStatus() status has no error message"); + return; + } + + if (strcmp(err_msg, _PyStatus_NO_MEMORY_ERRMSG) == 0) { + PyErr_NoMemory(); + return; + } + + const char *func = status.func; + if (func) { + PyErr_Format(PyExc_RuntimeError, "%s: %s", func, err_msg); } else { - PyErr_Format(PyExc_ValueError, "%s", status.err_msg); + PyErr_Format(PyExc_RuntimeError, "%s", err_msg); } - return NULL; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ee5d4981da51c8..64c74f433f222c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -629,10 +629,12 @@ pycore_create_interpreter(_PyRuntimeState *runtime, PyThreadState **tstate_p) { PyStatus status; - PyInterpreterState *interp = PyInterpreterState_New(); - if (interp == NULL) { - return _PyStatus_ERR("can't make main interpreter"); + PyInterpreterState *interp; + status = _PyInterpreterState_New(NULL, &interp); + if (_PyStatus_EXCEPTION(status)) { + return status; } + assert(interp != NULL); assert(_Py_IsMainInterpreter(interp)); status = _PyConfig_Copy(&interp->config, src_config); diff --git a/Python/pystate.c b/Python/pystate.c index 4a8808f700eaa2..46ea2c4f678db8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -426,13 +426,11 @@ init_runtime(_PyRuntimeState *runtime, Py_ssize_t unicode_next_index, PyThread_type_lock locks[NUMLOCKS]) { - if (runtime->_initialized) { - Py_FatalError("runtime already initialized"); - } - assert(!runtime->preinitializing && - !runtime->preinitialized && - !runtime->core_initialized && - !runtime->initialized); + assert(!runtime->preinitializing); + assert(!runtime->preinitialized); + assert(!runtime->core_initialized); + assert(!runtime->initialized); + assert(!runtime->_initialized); runtime->open_code_hook = open_code_hook; runtime->open_code_userdata = open_code_userdata; @@ -476,6 +474,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. memcpy(runtime, &initial, sizeof(*runtime)); + assert(!runtime->_initialized); } if (gilstate_tss_init(runtime) != 0) { @@ -647,14 +646,14 @@ free_interpreter(PyInterpreterState *interp) main interpreter. We fix those fields here, in addition to the other dynamically initialized fields. */ -static void +static PyStatus init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, PyInterpreterState *next, PyThread_type_lock pending_lock) { if (interp->_initialized) { - Py_FatalError("interpreter already initialized"); + return _PyStatus_ERR("interpreter already initialized"); } assert(runtime != NULL); @@ -675,7 +674,10 @@ init_interpreter(PyInterpreterState *interp, memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } - _PyObject_InitState(interp); + PyStatus status = _PyObject_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); @@ -701,42 +703,44 @@ init_interpreter(PyInterpreterState *interp, } interp->f_opcode_trace_set = false; interp->_initialized = 1; + return _PyStatus_OK(); } -PyInterpreterState * -PyInterpreterState_New(void) + +PyStatus +_PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) { - PyInterpreterState *interp; + *pinterp = NULL; + + // Don't get runtime from tstate since tstate can be NULL _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); - /* tstate is NULL when Py_InitializeFromConfig() calls - PyInterpreterState_New() to create the main interpreter. */ - if (_PySys_Audit(tstate, "cpython.PyInterpreterState_New", NULL) < 0) { - return NULL; + // tstate is NULL when pycore_create_interpreter() calls + // _PyInterpreterState_New() to create the main interpreter. + if (tstate != NULL) { + if (_PySys_Audit(tstate, "cpython.PyInterpreterState_New", NULL) < 0) { + return _PyStatus_ERR("sys.audit failed"); + } } PyThread_type_lock pending_lock = PyThread_allocate_lock(); if (pending_lock == NULL) { - if (tstate != NULL) { - _PyErr_NoMemory(tstate); - } - return NULL; + return _PyStatus_NO_MEMORY(); } - /* Don't get runtime from tstate since tstate can be NULL. */ - struct pyinterpreters *interpreters = &runtime->interpreters; - /* We completely serialize creation of multiple interpreters, since it simplifies things here and blocking concurrent calls isn't a problem. Regardless, we must fully block subinterpreter creation until after the main interpreter is created. */ HEAD_LOCK(runtime); + struct pyinterpreters *interpreters = &runtime->interpreters; int64_t id = interpreters->next_id; interpreters->next_id += 1; // Allocate the interpreter and add it to the runtime state. + PyInterpreterState *interp; + PyStatus status; PyInterpreterState *old_head = interpreters->head; if (old_head == NULL) { // We are creating the main interpreter. @@ -755,36 +759,59 @@ PyInterpreterState_New(void) interp = alloc_interpreter(); if (interp == NULL) { + status = _PyStatus_NO_MEMORY(); goto error; } // Set to _PyInterpreterState_INIT. - memcpy(interp, &initial._main_interpreter, - sizeof(*interp)); + memcpy(interp, &initial._main_interpreter, sizeof(*interp)); if (id < 0) { /* overflow or Py_Initialize() not called yet! */ - if (tstate != NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "failed to get an interpreter ID"); - } + status = _PyStatus_ERR("failed to get an interpreter ID"); goto error; } } interpreters->head = interp; - init_interpreter(interp, runtime, id, old_head, pending_lock); + status = init_interpreter(interp, runtime, + id, old_head, pending_lock); + if (_PyStatus_EXCEPTION(status)) { + goto error; + } + pending_lock = NULL; HEAD_UNLOCK(runtime); - return interp; + + assert(interp != NULL); + *pinterp = interp; + return _PyStatus_OK(); error: HEAD_UNLOCK(runtime); - PyThread_free_lock(pending_lock); + if (pending_lock != NULL) { + PyThread_free_lock(pending_lock); + } if (interp != NULL) { free_interpreter(interp); } - return NULL; + return status; +} + + +PyInterpreterState * +PyInterpreterState_New(void) +{ + // tstate can be NULL + PyThreadState *tstate = current_fast_get(&_PyRuntime); + + PyInterpreterState *interp; + PyStatus status = _PyInterpreterState_New(tstate, &interp); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + assert(interp != NULL); + return interp; } From e775601ef1ada83a80e1539d0bcd5509deadee14 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 1 Sep 2023 13:44:43 +0300 Subject: [PATCH 438/598] gh-101100: Fix sphinx warnings in `tutorial/classes.rst` (#108746) --- Doc/tools/.nitignore | 1 - Doc/tutorial/classes.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 22df73ef0cfb06..67641f73e4649c 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -156,7 +156,6 @@ Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst Doc/tutorial/appendix.rst -Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 91a3b73d2b55aa..7b92e1a51b6e67 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -276,7 +276,7 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`__doc__` is also a valid attribute, returning the docstring belonging to +:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class From 3047f09490ae63f25d57efe1d14a9a65d9b5f6db Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 1 Sep 2023 13:57:34 +0300 Subject: [PATCH 439/598] gh-101100: Fix sphinx warnings in `tutorial/appendix.rst` (#108750) Co-authored-by: Hugo van Kemenade --- Doc/tools/.nitignore | 1 - Doc/tutorial/appendix.rst | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 67641f73e4649c..1d89c9c683a070 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -155,7 +155,6 @@ Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst -Doc/tutorial/appendix.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 241a812037469e..588591fcdb726f 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -101,8 +101,8 @@ in the script:: The Customization Modules ------------------------- -Python provides two hooks to let you customize it: :mod:`sitecustomize` and -:mod:`usercustomize`. To see how it works, you need first to find the location +Python provides two hooks to let you customize it: :index:`sitecustomize` and +:index:`usercustomize`. To see how it works, you need first to find the location of your user site-packages directory. Start Python and run this code:: >>> import site @@ -113,9 +113,9 @@ Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless it is started with the :option:`-s` option to disable the automatic import. -:mod:`sitecustomize` works in the same way, but is typically created by an +:index:`sitecustomize` works in the same way, but is typically created by an administrator of the computer in the global site-packages directory, and is -imported before :mod:`usercustomize`. See the documentation of the :mod:`site` +imported before :index:`usercustomize`. See the documentation of the :mod:`site` module for more details. From 23f54c120067c96973619b9501fe4dff1b055188 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 14:59:09 +0200 Subject: [PATCH 440/598] Make test_fcntl quiet (#108758) Running test_fcntl logs two "struct.pack: ..." lines because multiprocessing imports test_fcntl twice with test.support.verbose=1. Move get_lockdata() inside TestFcntl test case and only call it where it's needed, to stop logging these lines. --- Lib/test/test_fcntl.py | 63 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 5da75615b41d79..203dd6fe57dcd9 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -16,37 +16,6 @@ -def get_lockdata(): - try: - os.O_LARGEFILE - except AttributeError: - start_len = "ll" - else: - start_len = "qq" - - if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): - if struct.calcsize('l') == 8: - off_t = 'l' - pid_t = 'i' - else: - off_t = 'lxxxx' - pid_t = 'l' - lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0, - fcntl.F_WRLCK, 0) - elif sys.platform.startswith('gnukfreebsd'): - lockdata = struct.pack('qqihhi', 0, 0, 0, fcntl.F_WRLCK, 0, 0) - elif sys.platform in ['hp-uxB', 'unixware7']: - lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) - else: - lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) - if lockdata: - if verbose: - print('struct.pack: ', repr(lockdata)) - return lockdata - -lockdata = get_lockdata() - class BadFile: def __init__(self, fn): self.fn = fn @@ -78,12 +47,43 @@ def tearDown(self): self.f.close() unlink(TESTFN) + @staticmethod + def get_lockdata(): + try: + os.O_LARGEFILE + except AttributeError: + start_len = "ll" + else: + start_len = "qq" + + if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) + or sys.platform == 'darwin'): + if struct.calcsize('l') == 8: + off_t = 'l' + pid_t = 'i' + else: + off_t = 'lxxxx' + pid_t = 'l' + lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0, + fcntl.F_WRLCK, 0) + elif sys.platform.startswith('gnukfreebsd'): + lockdata = struct.pack('qqihhi', 0, 0, 0, fcntl.F_WRLCK, 0, 0) + elif sys.platform in ['hp-uxB', 'unixware7']: + lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) + else: + lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) + if lockdata: + if verbose: + print('struct.pack: ', repr(lockdata)) + return lockdata + def test_fcntl_fileno(self): # the example from the library docs self.f = open(TESTFN, 'wb') rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) if verbose: print('Status from fcntl with O_NONBLOCK: ', rv) + lockdata = self.get_lockdata() rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata) if verbose: print('String from fcntl with F_SETLKW: ', repr(rv)) @@ -95,6 +95,7 @@ def test_fcntl_file_descriptor(self): rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK) if verbose: print('Status from fcntl with O_NONBLOCK: ', rv) + lockdata = self.get_lockdata() rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata) if verbose: print('String from fcntl with F_SETLKW: ', repr(rv)) From 5948f562e031a4d345681e1b962da3e3083f85df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 16:35:39 +0200 Subject: [PATCH 441/598] gh-108765: Reformat Include/osdefs.h (#108766) --- Include/osdefs.h | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/Include/osdefs.h b/Include/osdefs.h index 3243944a1483e9..2599e87a9d7c4b 100644 --- a/Include/osdefs.h +++ b/Include/osdefs.h @@ -1,51 +1,57 @@ +// Operating system dependencies. +// +// Define constants: +// +// - ALTSEP +// - DELIM +// - MAXPATHLEN +// - SEP + #ifndef Py_OSDEFS_H #define Py_OSDEFS_H #ifdef __cplusplus extern "C" { #endif - -/* Operating system dependencies */ - #ifdef MS_WINDOWS -#define SEP L'\\' -#define ALTSEP L'/' -#define MAXPATHLEN 256 -#define DELIM L';' +# define SEP L'\\' +# define ALTSEP L'/' +# define MAXPATHLEN 256 +# define DELIM L';' #endif #ifdef __VXWORKS__ -#define DELIM L';' +# define DELIM L';' #endif /* Filename separator */ #ifndef SEP -#define SEP L'/' +# define SEP L'/' #endif /* Max pathname length */ #ifdef __hpux -#include -#include -#ifndef PATH_MAX -#define PATH_MAX MAXPATHLEN -#endif +# include +# include +# ifndef PATH_MAX +# define PATH_MAX MAXPATHLEN +# endif #endif #ifndef MAXPATHLEN -#if defined(PATH_MAX) && PATH_MAX > 1024 -#define MAXPATHLEN PATH_MAX -#else -#define MAXPATHLEN 1024 -#endif +# if defined(PATH_MAX) && PATH_MAX > 1024 +# define MAXPATHLEN PATH_MAX +# else +# define MAXPATHLEN 1024 +# endif #endif /* Search path entry delimiter */ #ifndef DELIM -#define DELIM L':' +# define DELIM L':' #endif #ifdef __cplusplus } #endif -#endif /* !Py_OSDEFS_H */ +#endif // !Py_OSDEFS_H From 03c5a685689fd4891d01caff8da01096fcf73d5a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 16:57:28 +0200 Subject: [PATCH 442/598] gh-108765: Reformat Include/pymacconfig.h (#108764) Replace "MacOSX" with "macOS". --- Include/pymacconfig.h | 146 ++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 77 deletions(-) diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 00459a03b980be..806e41955efd7f 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -1,90 +1,82 @@ -#ifndef PYMACCONFIG_H -#define PYMACCONFIG_H - /* - * This file moves some of the autoconf magic to compile-time - * when building on MacOSX. This is needed for building 4-way - * universal binaries and for 64-bit universal binaries because - * the values redefined below aren't configure-time constant but - * only compile-time constant in these scenarios. - */ +// This file moves some of the autoconf magic to compile-time when building on +// macOS. This is needed for building 4-way universal binaries and for 64-bit +// universal binaries because the values redefined below aren't configure-time +// constant but only compile-time constant in these scenarios. -#if defined(__APPLE__) +#ifndef PY_MACCONFIG_H +#define PY_MACCONFIG_H +#ifdef __APPLE__ -# undef SIZEOF_LONG -# undef SIZEOF_PTHREAD_T -# undef SIZEOF_SIZE_T -# undef SIZEOF_TIME_T -# undef SIZEOF_VOID_P -# undef SIZEOF__BOOL -# undef SIZEOF_UINTPTR_T -# undef SIZEOF_PTHREAD_T -# undef WORDS_BIGENDIAN -# undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 -# undef DOUBLE_IS_BIG_ENDIAN_IEEE754 -# undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 -# undef HAVE_GCC_ASM_FOR_X87 +#undef SIZEOF_LONG +#undef SIZEOF_PTHREAD_T +#undef SIZEOF_SIZE_T +#undef SIZEOF_TIME_T +#undef SIZEOF_VOID_P +#undef SIZEOF__BOOL +#undef SIZEOF_UINTPTR_T +#undef SIZEOF_PTHREAD_T +#undef WORDS_BIGENDIAN +#undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 +#undef DOUBLE_IS_BIG_ENDIAN_IEEE754 +#undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 +#undef HAVE_GCC_ASM_FOR_X87 -# undef VA_LIST_IS_ARRAY -# if defined(__LP64__) && defined(__x86_64__) -# define VA_LIST_IS_ARRAY 1 -# endif - -# undef HAVE_LARGEFILE_SUPPORT -# ifndef __LP64__ -# define HAVE_LARGEFILE_SUPPORT 1 -# endif - -# undef SIZEOF_LONG -# ifdef __LP64__ -# define SIZEOF__BOOL 1 -# define SIZEOF__BOOL 1 -# define SIZEOF_LONG 8 -# define SIZEOF_PTHREAD_T 8 -# define SIZEOF_SIZE_T 8 -# define SIZEOF_TIME_T 8 -# define SIZEOF_VOID_P 8 -# define SIZEOF_UINTPTR_T 8 -# define SIZEOF_PTHREAD_T 8 -# else -# ifdef __ppc__ -# define SIZEOF__BOOL 4 -# else -# define SIZEOF__BOOL 1 -# endif -# define SIZEOF_LONG 4 -# define SIZEOF_PTHREAD_T 4 -# define SIZEOF_SIZE_T 4 -# define SIZEOF_TIME_T 4 -# define SIZEOF_VOID_P 4 -# define SIZEOF_UINTPTR_T 4 -# define SIZEOF_PTHREAD_T 4 -# endif +#undef VA_LIST_IS_ARRAY +#if defined(__LP64__) && defined(__x86_64__) +# define VA_LIST_IS_ARRAY 1 +#endif -# if defined(__LP64__) - /* MacOSX 10.4 (the first release to support 64-bit code - * at all) only supports 64-bit in the UNIX layer. - * Therefore suppress the toolbox-glue in 64-bit mode. - */ +#undef HAVE_LARGEFILE_SUPPORT +#ifndef __LP64__ +# define HAVE_LARGEFILE_SUPPORT 1 +#endif - /* In 64-bit mode setpgrp always has no arguments, in 32-bit - * mode that depends on the compilation environment - */ -# undef SETPGRP_HAVE_ARG +#undef SIZEOF_LONG +#ifdef __LP64__ +# define SIZEOF__BOOL 1 +# define SIZEOF__BOOL 1 +# define SIZEOF_LONG 8 +# define SIZEOF_PTHREAD_T 8 +# define SIZEOF_SIZE_T 8 +# define SIZEOF_TIME_T 8 +# define SIZEOF_VOID_P 8 +# define SIZEOF_UINTPTR_T 8 +# define SIZEOF_PTHREAD_T 8 +#else +# ifdef __ppc__ +# define SIZEOF__BOOL 4 +# else +# define SIZEOF__BOOL 1 +# endif +# define SIZEOF_LONG 4 +# define SIZEOF_PTHREAD_T 4 +# define SIZEOF_SIZE_T 4 +# define SIZEOF_TIME_T 4 +# define SIZEOF_VOID_P 4 +# define SIZEOF_UINTPTR_T 4 +# define SIZEOF_PTHREAD_T 4 +#endif -# endif +// macOS 10.4 (the first release to support 64-bit code +// at all) only supports 64-bit in the UNIX layer. +// Therefore suppress the toolbox-glue in 64-bit mode. +// +// In 64-bit mode setpgrp always has no arguments, in 32-bit +// mode that depends on the compilation environment +#if defined(__LP64__) +# undef SETPGRP_HAVE_ARG +#endif #ifdef __BIG_ENDIAN__ -#define WORDS_BIGENDIAN 1 -#define DOUBLE_IS_BIG_ENDIAN_IEEE754 +# define WORDS_BIGENDIAN 1 +# define DOUBLE_IS_BIG_ENDIAN_IEEE754 #else -#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 -#endif /* __BIG_ENDIAN */ +# define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 +#endif #ifdef __i386__ -# define HAVE_GCC_ASM_FOR_X87 +# define HAVE_GCC_ASM_FOR_X87 #endif - -#endif /* defined(_APPLE__) */ - -#endif /* PYMACCONFIG_H */ +#endif // __APPLE__ +#endif // !PY_MACCONFIG_H From 578ebc5d5fab2e5e0b87636361c6aeb8d2b287fa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 18:36:53 +0200 Subject: [PATCH 443/598] gh-108767: Replace ctype.h functions with pyctype.h functions (#108772) Replace locale dependent functions with Python "pyctype.h" locale independent functions: * Replace isalpha() with Py_ISALPHA(). * Replace isdigit() with Py_ISDIGIT(). * Replace isxdigit() with Py_ISXDIGIT(). * Replace tolower() with Py_TOLOWER(). Leave Modules/_sre/sre.c unchanged, it uses locale dependent functions on purpose. Include explicitly in _decimal.c to get isascii(). --- Modules/_decimal/_decimal.c | 1 + Modules/_zoneinfo.c | 14 +++++++------- Modules/getaddrinfo.c | 4 ++-- Objects/bytesobject.c | 8 ++++---- Parser/tokenizer.c | 37 ++++++++++++++++++------------------- Python/pystrcmp.c | 8 ++++---- 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 585214cc45d6cd..b49ea3cbb410ef 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -35,6 +35,7 @@ #include "complexobject.h" #include "mpdecimal.h" +#include // isascii() #include #include "docstrings.h" diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 09f5fd4b2ef0a9..3f7b2851c5b794 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -1701,7 +1701,7 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) static int parse_uint(const char *const p, uint8_t *value) { - if (!isdigit(*p)) { + if (!Py_ISDIGIT(*p)) { return -1; } @@ -1732,7 +1732,7 @@ parse_abbr(const char *const p, PyObject **abbr) // '+' ) character, or the minus-sign ( '-' ) character. The std // and dst fields in this case shall not include the quoting // characters. - if (!isalpha(buff) && !isdigit(buff) && buff != '+' && + if (!Py_ISALPHA(buff) && !Py_ISDIGIT(buff) && buff != '+' && buff != '-') { return -1; } @@ -1748,7 +1748,7 @@ parse_abbr(const char *const p, PyObject **abbr) // In the unquoted form, all characters in these fields shall be // alphabetic characters from the portable character set in the // current locale. - while (isalpha(*ptr)) { + while (Py_ISALPHA(*ptr)) { ptr++; } str_end = ptr; @@ -1802,7 +1802,7 @@ parse_tz_delta(const char *const p, long *total_seconds) // The hour can be 1 or 2 numeric characters for (size_t i = 0; i < 2; ++i) { buff = *ptr; - if (!isdigit(buff)) { + if (!Py_ISDIGIT(buff)) { if (i == 0) { return -1; } @@ -1830,7 +1830,7 @@ parse_tz_delta(const char *const p, long *total_seconds) for (size_t j = 0; j < 2; ++j) { buff = *ptr; - if (!isdigit(buff)) { + if (!Py_ISDIGIT(buff)) { return -1; } *(outputs[i]) *= 10; @@ -1932,7 +1932,7 @@ parse_transition_rule(const char *const p, TransitionRuleType **out) } for (size_t i = 0; i < 3; ++i) { - if (!isdigit(*ptr)) { + if (!Py_ISDIGIT(*ptr)) { if (i == 0) { return -1; } @@ -2007,7 +2007,7 @@ parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, uint8_t buff = 0; for (size_t j = 0; j < 2; j++) { - if (!isdigit(*ptr)) { + if (!Py_ISDIGIT(*ptr)) { if (i == 0 && j > 0) { break; } diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index f1c28d7d9312ac..6fb6062a6520d9 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -51,7 +51,6 @@ #include #include #include -#include #include #include "addrinfo.h" @@ -228,8 +227,9 @@ str_isnumber(const char *p) { unsigned char *q = (unsigned char *)p; while (*q) { - if (! isdigit(*q)) + if (!Py_ISDIGIT(*q)) { return NO; + } q++; } return YES; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index c3a31bec822c37..26227dd251122d 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -722,11 +722,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, if (--fmtcnt >= 0) c = *fmt++; } - else if (c >= 0 && isdigit(c)) { + else if (c >= 0 && Py_ISDIGIT(c)) { width = c - '0'; while (--fmtcnt >= 0) { c = Py_CHARMASK(*fmt++); - if (!isdigit(c)) + if (!Py_ISDIGIT(c)) break; if (width > (PY_SSIZE_T_MAX - ((int)c - '0')) / 10) { PyErr_SetString( @@ -761,11 +761,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, if (--fmtcnt >= 0) c = *fmt++; } - else if (c >= 0 && isdigit(c)) { + else if (c >= 0 && Py_ISDIGIT(c)) { prec = c - '0'; while (--fmtcnt >= 0) { c = Py_CHARMASK(*fmt++); - if (!isdigit(c)) + if (!Py_ISDIGIT(c)) break; if (prec > (INT_MAX - ((int)c - '0')) / 10) { PyErr_SetString( diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index b10c9f1f8ea2cc..6ec24895785568 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -4,7 +4,6 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() -#include #include #include "tokenizer.h" @@ -158,7 +157,7 @@ get_normal_name(const char *s) /* for utf-8 and latin-1 */ else if (c == '_') buf[i] = '-'; else - buf[i] = tolower(c); + buf[i] = Py_TOLOWER(c); } buf[i] = '\0'; if (strcmp(buf, "utf-8") == 0 || @@ -1715,12 +1714,12 @@ tok_decimal_tail(struct tok_state *tok) while (1) { do { c = tok_nextc(tok); - } while (isdigit(c)); + } while (Py_ISDIGIT(c)); if (c != '_') { break; } c = tok_nextc(tok); - if (!isdigit(c)) { + if (!Py_ISDIGIT(c)) { tok_backup(tok, c); syntaxerror(tok, "invalid decimal literal"); return 0; @@ -2108,7 +2107,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Period or number starting with period? */ if (c == '.') { c = tok_nextc(tok); - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { goto fraction; } else if (c == '.') { c = tok_nextc(tok); @@ -2131,7 +2130,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } /* Number */ - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { if (c == '0') { /* Hex, octal or binary -- maybe. */ c = tok_nextc(tok); @@ -2142,13 +2141,13 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (c == '_') { c = tok_nextc(tok); } - if (!isxdigit(c)) { + if (!Py_ISXDIGIT(c)) { tok_backup(tok, c); return MAKE_TOKEN(syntaxerror(tok, "invalid hexadecimal literal")); } do { c = tok_nextc(tok); - } while (isxdigit(c)); + } while (Py_ISXDIGIT(c)); } while (c == '_'); if (!verify_end_of_number(tok, c, "hexadecimal")) { return MAKE_TOKEN(ERRORTOKEN); @@ -2162,7 +2161,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); } if (c < '0' || c >= '8') { - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { return MAKE_TOKEN(syntaxerror(tok, "invalid digit '%c' in octal literal", c)); } @@ -2175,7 +2174,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); } while ('0' <= c && c < '8'); } while (c == '_'); - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { return MAKE_TOKEN(syntaxerror(tok, "invalid digit '%c' in octal literal", c)); } @@ -2191,7 +2190,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); } if (c != '0' && c != '1') { - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { return MAKE_TOKEN(syntaxerror(tok, "invalid digit '%c' in binary literal", c)); } else { @@ -2203,7 +2202,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); } while (c == '0' || c == '1'); } while (c == '_'); - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { return MAKE_TOKEN(syntaxerror(tok, "invalid digit '%c' in binary literal", c)); } if (!verify_end_of_number(tok, c, "binary")) { @@ -2217,7 +2216,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t while (1) { if (c == '_') { c = tok_nextc(tok); - if (!isdigit(c)) { + if (!Py_ISDIGIT(c)) { tok_backup(tok, c); return MAKE_TOKEN(syntaxerror(tok, "invalid decimal literal")); } @@ -2228,7 +2227,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); } char* zeros_end = tok->cur; - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { nonzero = 1; c = tok_decimal_tail(tok); if (c == 0) { @@ -2272,7 +2271,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); fraction: /* Fraction */ - if (isdigit(c)) { + if (Py_ISDIGIT(c)) { c = tok_decimal_tail(tok); if (c == 0) { return MAKE_TOKEN(ERRORTOKEN); @@ -2287,11 +2286,11 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); if (c == '+' || c == '-') { c = tok_nextc(tok); - if (!isdigit(c)) { + if (!Py_ISDIGIT(c)) { tok_backup(tok, c); return MAKE_TOKEN(syntaxerror(tok, "invalid decimal literal")); } - } else if (!isdigit(c)) { + } else if (!Py_ISDIGIT(c)) { tok_backup(tok, c); if (!verify_end_of_number(tok, e, "decimal")) { return MAKE_TOKEN(ERRORTOKEN); @@ -2326,7 +2325,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } f_string_quote: - if (((tolower(*tok->start) == 'f' || tolower(*tok->start) == 'r') && (c == '\'' || c == '"'))) { + if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r') && (c == '\'' || c == '"'))) { int quote = c; int quote_size = 1; /* 1 or 3 */ @@ -2377,7 +2376,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t switch (*tok->start) { case 'F': case 'f': - the_current_tok->f_string_raw = tolower(*(tok->start + 1)) == 'r'; + the_current_tok->f_string_raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; break; case 'R': case 'r': diff --git a/Python/pystrcmp.c b/Python/pystrcmp.c index 9224ce4c706055..9796cb013ad450 100644 --- a/Python/pystrcmp.c +++ b/Python/pystrcmp.c @@ -11,11 +11,11 @@ PyOS_mystrnicmp(const char *s1, const char *s2, Py_ssize_t size) return 0; p1 = (const unsigned char *)s1; p2 = (const unsigned char *)s2; - for (; (--size > 0) && *p1 && *p2 && (tolower(*p1) == tolower(*p2)); + for (; (--size > 0) && *p1 && *p2 && (Py_TOLOWER(*p1) == Py_TOLOWER(*p2)); p1++, p2++) { ; } - return tolower(*p1) - tolower(*p2); + return Py_TOLOWER(*p1) - Py_TOLOWER(*p2); } int @@ -23,8 +23,8 @@ PyOS_mystricmp(const char *s1, const char *s2) { const unsigned char *p1 = (const unsigned char *)s1; const unsigned char *p2 = (const unsigned char *)s2; - for (; *p1 && *p2 && (tolower(*p1) == tolower(*p2)); p1++, p2++) { + for (; *p1 && *p2 && (Py_TOLOWER(*p1) == Py_TOLOWER(*p2)); p1++, p2++) { ; } - return (tolower(*p1) - tolower(*p2)); + return (Py_TOLOWER(*p1) - Py_TOLOWER(*p2)); } From 0e01fac315dfa705ac8a6954485546f28cf4c87d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 1 Sep 2023 21:42:42 +0300 Subject: [PATCH 444/598] Add Modules/_testcapi/util.h header (GH-108774) It contains common macros used in C API tests. --- Makefile.pre.in | 2 +- Modules/_testcapi/abstract.c | 19 +---------- Modules/_testcapi/dict.c | 19 +---------- Modules/_testcapi/exceptions.c | 2 +- Modules/_testcapi/unicode.c | 58 +++++++++++----------------------- Modules/_testcapi/util.h | 25 +++++++++++++++ Modules/_testcapimodule.c | 18 ++--------- 7 files changed, 49 insertions(+), 94 deletions(-) create mode 100644 Modules/_testcapi/util.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 0870cb3d899d45..5826b9a82736b9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2799,7 +2799,7 @@ MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_ MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h -MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h +MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index 10d7ff8d4a7bc8..91a1ee3aaafc02 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -1,24 +1,7 @@ #include // ptrdiff_t #include "parts.h" - -#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); - -#define RETURN_INT(value) do { \ - int _ret = (value); \ - if (_ret == -1) { \ - return NULL; \ - } \ - return PyLong_FromLong(_ret); \ - } while (0) - -#define RETURN_SIZE(value) do { \ - Py_ssize_t _ret = (value); \ - if (_ret == -1) { \ - return NULL; \ - } \ - return PyLong_FromSsize_t(_ret); \ - } while (0) +#include "util.h" static PyObject * diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 6c3f9cd22faa5a..6720f0437401ef 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -1,24 +1,7 @@ #include // ptrdiff_t #include "parts.h" - -#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); - -#define RETURN_INT(value) do { \ - int _ret = (value); \ - if (_ret == -1) { \ - return NULL; \ - } \ - return PyLong_FromLong(_ret); \ - } while (0) - -#define RETURN_SIZE(value) do { \ - Py_ssize_t _ret = (value); \ - if (_ret == -1) { \ - return NULL; \ - } \ - return PyLong_FromSsize_t(_ret); \ - } while (0) +#include "util.h" static PyObject * diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index d7edf8c7ce6744..025b42db247e81 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -1,7 +1,7 @@ #include "parts.h" +#include "util.h" #include "clinic/exceptions.c.h" -#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); /*[clinic input] module _testcapi diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index b4c6c4b5e3ce1a..f2cf9a744007d9 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1,6 +1,7 @@ #include // ptrdiff_t #include "parts.h" +#include "util.h" static struct PyModuleDef *_testcapimodule = NULL; // set at initialization @@ -101,7 +102,6 @@ test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); static PyObject * unicode_copy(PyObject *unicode) @@ -347,13 +347,8 @@ unicode_substring(PyObject *self, PyObject *args) static PyObject * unicode_getlength(PyObject *self, PyObject *arg) { - Py_ssize_t result; - NULLABLE(arg); - result = PyUnicode_GetLength(arg); - if (result == -1) - return NULL; - return PyLong_FromSsize_t(result); + RETURN_SIZE(PyUnicode_GetLength(arg)); } /* Test PyUnicode_ReadChar() */ @@ -482,16 +477,12 @@ static PyObject * unicode_aswidechar_null(PyObject *self, PyObject *args) { PyObject *unicode; - Py_ssize_t buflen, size; + Py_ssize_t buflen; if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; NULLABLE(unicode); - size = PyUnicode_AsWideChar(unicode, NULL, buflen); - if (size == -1) { - return NULL; - } - return PyLong_FromSsize_t(size); + RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); } /* Test PyUnicode_AsWideCharString() */ @@ -1293,17 +1284,13 @@ unicode_count(PyObject *self, PyObject *args) PyObject *substr; Py_ssize_t start; Py_ssize_t end; - Py_ssize_t result; if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) return NULL; NULLABLE(str); NULLABLE(substr); - result = PyUnicode_Count(str, substr, start, end); - if (result == -1) - return NULL; - return PyLong_FromSsize_t(result); + RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); } /* Test PyUnicode_Find() */ @@ -1323,8 +1310,11 @@ unicode_find(PyObject *self, PyObject *args) NULLABLE(str); NULLABLE(substr); result = PyUnicode_Find(str, substr, start, end, direction); - if (result == -2) + if (result == -2) { + assert(PyErr_Occurred()); return NULL; + } + assert(!PyErr_Occurred()); return PyLong_FromSsize_t(result); } @@ -1337,17 +1327,13 @@ unicode_tailmatch(PyObject *self, PyObject *args) Py_ssize_t start; Py_ssize_t end; int direction; - Py_ssize_t result; if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) return NULL; NULLABLE(str); NULLABLE(substr); - result = PyUnicode_Tailmatch(str, substr, start, end, direction); - if (result == -1) - return NULL; - return PyLong_FromSsize_t(result); + RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); } /* Test PyUnicode_FindChar() */ @@ -1366,10 +1352,12 @@ unicode_findchar(PyObject *self, PyObject *args) } NULLABLE(str); result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); - if (result == -2) + if (result == -2) { + assert(PyErr_Occurred()); return NULL; - else - return PyLong_FromSsize_t(result); + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); } /* Test PyUnicode_Replace() */ @@ -1407,6 +1395,7 @@ unicode_compare(PyObject *self, PyObject *args) if (result == -1 && PyErr_Occurred()) { return NULL; } + assert(!PyErr_Occurred()); return PyLong_FromLong(result); } @@ -1467,32 +1456,21 @@ unicode_contains(PyObject *self, PyObject *args) { PyObject *container; PyObject *element; - int result; if (!PyArg_ParseTuple(args, "OO", &container, &element)) return NULL; NULLABLE(container); NULLABLE(element); - result = PyUnicode_Contains(container, element); - if (result == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromLong(result); + RETURN_INT(PyUnicode_Contains(container, element)); } /* Test PyUnicode_IsIdentifier() */ static PyObject * unicode_isidentifier(PyObject *self, PyObject *arg) { - int result; - NULLABLE(arg); - result = PyUnicode_IsIdentifier(arg); - if (result == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromLong(result); + RETURN_INT(PyUnicode_IsIdentifier(arg)); } /* Test PyUnicode_CopyCharacters() */ diff --git a/Modules/_testcapi/util.h b/Modules/_testcapi/util.h new file mode 100644 index 00000000000000..05ccb12a4444f8 --- /dev/null +++ b/Modules/_testcapi/util.h @@ -0,0 +1,25 @@ +#define NULLABLE(x) do { \ + if (x == Py_None) { \ + x = NULL; \ + } \ + } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromSsize_t(_ret); \ + } while (0) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f8b36ff9ce835a..4fc354ae79bfed 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -43,16 +43,8 @@ // Several parts of this module are broken out into files in _testcapi/. // Include definitions from there. #include "_testcapi/parts.h" +#include "_testcapi/util.h" -#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); - -#define RETURN_INT(value) do { \ - int _ret = (value); \ - if (_ret == -1) { \ - return NULL; \ - } \ - return PyLong_FromLong(_ret); \ - } while (0) // Forward declarations static struct PyModuleDef _testcapimodule; @@ -1333,19 +1325,13 @@ static PyObject * test_PyBuffer_SizeFromFormat(PyObject *self, PyObject *args) { const char *format; - Py_ssize_t result; if (!PyArg_ParseTuple(args, "s:test_PyBuffer_SizeFromFormat", &format)) { return NULL; } - result = PyBuffer_SizeFromFormat(format); - if (result == -1) { - return NULL; - } - - return PyLong_FromSsize_t(result); + RETURN_SIZE(PyBuffer_SizeFromFormat(format)); } /* Test that the fatal error from not having a current thread doesn't From 45b9e6a61f4cd61c08b2b02d257b52635ab37a25 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Sep 2023 21:03:20 +0200 Subject: [PATCH 445/598] gh-108765: Move standard includes to Python.h (#108769) * Move , and standard includes to Python.h. * Move "pystats.h" include from object.h to Python.h. * Remove redundant "pymem.h" include in objimpl.h and "pyport.h" include in pymem.h; Python.h already includes them earlier. * Remove redundant include in unicodeobject.h; Python.h already includes it. * Move _SGI_MP_SOURCE define from Python.h to pyport.h. * pycore_condvar.h includes explicitly for the _POSIX_THREADS macro. --- Include/Python.h | 40 ++++++++++++++++++------------- Include/bytesobject.h | 5 +--- Include/internal/pycore_condvar.h | 4 ++++ Include/modsupport.h | 5 +--- Include/object.h | 2 -- Include/objimpl.h | 10 +++----- Include/pyerrors.h | 6 ++--- Include/pymem.h | 11 +++------ Include/pyport.h | 11 ++++----- Include/unicodeobject.h | 8 ------- 10 files changed, 43 insertions(+), 59 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 07f6c202a7f126..44f0fd3ee4b56b 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -5,42 +5,50 @@ #ifndef Py_PYTHON_H #define Py_PYTHON_H -// Since this is a "meta-include" file, no #ifdef __cplusplus / extern "C" { +// Since this is a "meta-include" file, "#ifdef __cplusplus / extern "C" {" +// is not needed. + // Include Python header files #include "patchlevel.h" #include "pyconfig.h" #include "pymacconfig.h" -#if defined(__sgi) && !defined(_SGI_MP_SOURCE) -# define _SGI_MP_SOURCE + +// Include standard header files +#include // assert() +#include // tolower() +#include // uintptr_t +#include // INT_MAX +#include // va_list +#include // wchar_t +#ifdef HAVE_STDDEF_H +# include // size_t +#endif +#ifndef MS_WINDOWS +# include // sysconf() #endif -// stdlib.h, stdio.h, errno.h and string.h headers are not used by Python -// headers, but kept for backward compatibility. They are excluded from the -// limited C API of Python 3.11. +// errno.h, stdio.h, stdlib.h and string.h headers are no longer used by Python +// headers, but kept for backward compatibility (no introduce new compiler +// warnings). They are not included by the limited C API version 3.11 and +// above. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# include -# include // FILE* # include // errno +# include // FILE* +# include // getenv() # include // memcpy() #endif -#ifndef MS_WINDOWS -# include -#endif -#ifdef HAVE_STDDEF_H -# include // size_t -#endif -#include // assert() -#include // wchar_t +// Include Python header files #include "pyport.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" #include "pytypedefs.h" #include "pybuffer.h" +#include "pystats.h" #include "object.h" #include "objimpl.h" #include "typeslots.h" diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ee448cd02bdab3..c5a24195be6bc3 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -1,5 +1,4 @@ - -/* Bytes object interface */ +// Bytes object interface #ifndef Py_BYTESOBJECT_H #define Py_BYTESOBJECT_H @@ -7,8 +6,6 @@ extern "C" { #endif -#include // va_list - /* Type PyBytesObject represents a byte string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index db8682a4c077aa..489e67d4ec4f9f 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,6 +5,10 @@ # error "this header requires Py_BUILD_CORE define" #endif +#ifndef MS_WINDOWS +# include // _POSIX_THREADS +#endif + #ifndef _POSIX_THREADS /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external diff --git a/Include/modsupport.h b/Include/modsupport.h index 7c15ab50c320e2..6efe9dfaa9089e 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -1,3 +1,4 @@ +// Module support interface #ifndef Py_MODSUPPORT_H #define Py_MODSUPPORT_H @@ -5,10 +6,6 @@ extern "C" { #endif -/* Module support interface */ - -#include // va_list - PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, diff --git a/Include/object.h b/Include/object.h index de2a1ce0f3c4dd..b94b2907e4f163 100644 --- a/Include/object.h +++ b/Include/object.h @@ -51,8 +51,6 @@ A standard interface exists for objects that contain an array of items whose size is determined when the object is allocated. */ -#include "pystats.h" - /* Py_DEBUG implies Py_REF_DEBUG. */ #if defined(Py_DEBUG) && !defined(Py_REF_DEBUG) # define Py_REF_DEBUG diff --git a/Include/objimpl.h b/Include/objimpl.h index ef871c5ea93ebe..967e2776767756 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -1,12 +1,8 @@ -/* The PyObject_ memory family: high-level object memory interfaces. - See pymem.h for the low-level PyMem_ family. -*/ +// The PyObject_ memory family: high-level object memory interfaces. +// See pymem.h for the low-level PyMem_ family. #ifndef Py_OBJIMPL_H #define Py_OBJIMPL_H - -#include "pymem.h" - #ifdef __cplusplus extern "C" { #endif @@ -231,4 +227,4 @@ PyAPI_FUNC(int) PyObject_GC_IsFinalized(PyObject *); #ifdef __cplusplus } #endif -#endif /* !Py_OBJIMPL_H */ +#endif // !Py_OBJIMPL_H diff --git a/Include/pyerrors.h b/Include/pyerrors.h index d089fa71779330..5d0028c116e2d8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -1,13 +1,11 @@ +// Error handling definitions + #ifndef Py_ERRORS_H #define Py_ERRORS_H #ifdef __cplusplus extern "C" { #endif -#include // va_list - -/* Error handling definitions */ - PyAPI_FUNC(void) PyErr_SetNone(PyObject *); PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *); PyAPI_FUNC(void) PyErr_SetString( diff --git a/Include/pymem.h b/Include/pymem.h index e882645757bfd3..68e33f90b7b913 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -1,12 +1,8 @@ -/* The PyMem_ family: low-level memory allocation interfaces. - See objimpl.h for the PyObject_ memory family. -*/ +// The PyMem_ family: low-level memory allocation interfaces. +// See objimpl.h for the PyObject_ memory family. #ifndef Py_PYMEM_H #define Py_PYMEM_H - -#include "pyport.h" - #ifdef __cplusplus extern "C" { #endif @@ -100,5 +96,4 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifdef __cplusplus } #endif - -#endif /* !Py_PYMEM_H */ +#endif // !Py_PYMEM_H diff --git a/Include/pyport.h b/Include/pyport.h index 115b54fd969287..511c3fda1a40a5 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -1,13 +1,8 @@ #ifndef Py_PYPORT_H #define Py_PYPORT_H -#include "pyconfig.h" /* include for defines */ - -#include - -#include #ifndef UCHAR_MAX -# error "limits.h must define UCHAR_MAX" +# error " header must define UCHAR_MAX" #endif #if UCHAR_MAX != 255 # error "Python's source code assumes C's unsigned char is an 8-bit type" @@ -771,4 +766,8 @@ extern char * _getpty(int *, int, mode_t, int); # define ALIGNOF_MAX_ALIGN_T _Alignof(long double) #endif +#if defined(__sgi) && !defined(_SGI_MP_SOURCE) +# define _SGI_MP_SOURCE +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 5839c747a29275..f00277787122aa 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1,8 +1,6 @@ #ifndef Py_UNICODEOBJECT_H #define Py_UNICODEOBJECT_H -#include // va_list - /* Unicode implementation based on original code by Fredrik Lundh, @@ -55,8 +53,6 @@ Copyright (c) Corporation for National Research Initiatives. * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * -------------------------------------------------------------------- */ -#include - /* === Internal API ======================================================= */ /* --- Internal Unicode Format -------------------------------------------- */ @@ -93,10 +89,6 @@ Copyright (c) Corporation for National Research Initiatives. # endif #endif -#ifdef HAVE_WCHAR_H -# include -#endif - /* Py_UCS4 and Py_UCS2 are typedefs for the respective unicode representations. */ typedef uint32_t Py_UCS4; From 6f97eeec222f81bd7ae836c149872a40b079e2a6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 1 Sep 2023 22:21:39 +0300 Subject: [PATCH 446/598] Improve some C API documentation (GH-108768) * Express functions which take argument as a C string in terms of functions which take Python object. * Use "note" directive for PyMapping_HasKey() and PyMapping_HasKeyString() notes. --- Doc/c-api/dict.rst | 16 +++++------- Doc/c-api/mapping.rst | 57 +++++++++++++++++++++---------------------- Doc/c-api/object.rst | 40 +++++++++++++----------------- 3 files changed, 51 insertions(+), 62 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 7810d05b0ebc2e..8ee351918006e4 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -79,12 +79,9 @@ Dictionary Objects .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) - .. index:: single: PyUnicode_FromString() - - Insert *val* into the dictionary *p* using *key* as a key. *key* should - be a :c:expr:`const char*` UTF-8 encoded bytes string. The key object is created using - ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on - failure. This function *does not* steal a reference to *val*. + This is the same as :c:func:`PyDict_SetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) @@ -97,10 +94,9 @@ Dictionary Objects .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - Remove the entry in dictionary *p* which has a key specified by the UTF-8 - encoded bytes string *key*. - If *key* is not in the dictionary, :exc:`KeyError` is raised. - Return ``0`` on success or ``-1`` on failure. + This is the same as :c:func:`PyDict_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyDict_GetItemRef(PyObject *p, PyObject *key, PyObject **result) diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 82011d9d36a524..5b909ef8f5e2ce 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -28,9 +28,9 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - Return element of *o* corresponding to the string *key* or ``NULL`` on failure. - This is the equivalent of the Python expression ``o[key]``. - See also :c:func:`PyObject_GetItem`. + This is the same as :c:func:`PyObject_GetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) @@ -50,38 +50,30 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: int PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) - Variant of :c:func:`PyMapping_GetItemString` which doesn't raise - :exc:`KeyError` if the key is not found. - - If the key is found, return ``1`` and set *\*result* to a new - :term:`strong reference` to the corresponding value. - If the key is not found, return ``0`` and set *\*result* to ``NULL``; - the :exc:`KeyError` is silenced. - If an error other than :exc:`KeyError` is raised, return ``-1`` and - set *\*result* to ``NULL``. + This is the same as :c:func:`PyMapping_GetOptionalItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. versionadded:: 3.13 .. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) - Map the string *key* to the value *v* in object *o*. Returns ``-1`` on - failure. This is the equivalent of the Python statement ``o[key] = v``. - See also :c:func:`PyObject_SetItem`. This function *does not* steal a - reference to *v*. + This is the same as :c:func:`PyObject_SetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) - Remove the mapping for the object *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. This is an alias of :c:func:`PyObject_DelItem`. .. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) - Remove the mapping for the string *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) @@ -90,20 +82,27 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and This is equivalent to the Python expression ``key in o``. This function always succeeds. - Note that exceptions which occur while calling the :meth:`~object.__getitem__` - method will get suppressed. - To get error reporting use :c:func:`PyObject_GetItem()` instead. + .. note:: + + Exceptions which occur when this calls :meth:`~object.__getitem__` + method are silently ignored. + For proper error handling, use :c:func:`PyMapping_GetOptionalItem` or + :c:func:`PyObject_GetItem()` instead. .. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) - Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. - This is equivalent to the Python expression ``key in o``. - This function always succeeds. + This is the same as :c:func:`PyMapping_HasKey`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. note:: - Note that exceptions which occur while calling the :meth:`~object.__getitem__` - method and creating a temporary string object will get suppressed. - To get error reporting use :c:func:`PyMapping_GetItemString()` instead. + Exceptions that occur when this calls :meth:`~object.__getitem__` + method or while creating the temporary :class:`str` + object are silently ignored. + For proper error handling, use :c:func:`PyMapping_GetOptionalItemString` or + :c:func:`PyMapping_GetItemString` instead. .. c:function:: PyObject* PyMapping_Keys(PyObject *o) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 6d9fc7f50604da..2572c087738fe3 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -43,15 +43,15 @@ Object Protocol .. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. + This is the same as :c:func:`PyObject_HasAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods or while creating the temporary :class:`str` - object are silently ignored. + :meth:`~object.__getattribute__` methods or while creating the temporary + :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyObject_GetOptionalAttrString` or :c:func:`PyObject_GetAttrString` instead. @@ -68,9 +68,9 @@ Object Protocol .. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) - Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or ``NULL`` on failure. This is the equivalent of the Python - expression ``o.attr_name``. + This is the same as :c:func:`PyObject_GetAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. If the missing attribute should not be treated as a failure, you can use :c:func:`PyObject_GetOptionalAttrString` instead. @@ -93,15 +93,9 @@ Object Protocol .. c:function:: int PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result); - Variant of :c:func:`PyObject_GetAttrString` which doesn't raise - :exc:`AttributeError` if the attribute is not found. - - If the attribute is found, return ``1`` and set *\*result* to a new - :term:`strong reference` to the attribute. - If the attribute is not found, return ``0`` and set *\*result* to ``NULL``; - the :exc:`AttributeError` is silenced. - If an error other than :exc:`AttributeError` is raised, return ``-1`` and - set *\*result* to ``NULL``. + This is the same as :c:func:`PyObject_GetOptionalAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. versionadded:: 3.13 @@ -129,10 +123,9 @@ Object Protocol .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) - Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement - ``o.attr_name = v``. + This is the same as :c:func:`PyObject_SetAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. @@ -158,8 +151,9 @@ Object Protocol .. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name) - Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. - This is the equivalent of the Python statement ``del o.attr_name``. + This is the same as :c:func:`PyObject_DelAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) From 8f9ea43ee805f98391f857397daac9df7ffa71cd Mon Sep 17 00:00:00 2001 From: TATHAGATA ROY Date: Sat, 2 Sep 2023 02:06:24 +0530 Subject: [PATCH 447/598] gh-105563: reference DateType in datetime's documentation (#105946) --- Doc/library/datetime.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 400c369a9bb736..04cc75562937e0 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -37,6 +37,10 @@ on efficient attribute extraction for output formatting and manipulation. Package `dateutil `_ Third-party library with expanded time zone and parsing support. + Package `DateType `_ + Third-party library that introduces distinct static types to e.g. allow static type checkers + to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: Aware and Naive Objects From a1cbace91b624681dd7bb8eb82ba187bda55d785 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 1 Sep 2023 21:57:25 +0100 Subject: [PATCH 448/598] gh-105509: Simplify implementation of `typing.Annotated` (#105510) --- Lib/test/test_typing.py | 3 +- Lib/typing.py | 38 +++++++------------ ...-06-08-15-56-45.gh-issue-105509.YIG57j.rst | 3 ++ 3 files changed, 17 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-06-08-15-56-45.gh-issue-105509.YIG57j.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 6447ed6078e178..69f5ff913c57bb 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -8189,8 +8189,7 @@ class AnnotatedTests(BaseTestCase): def test_new(self): with self.assertRaisesRegex( - TypeError, - 'Type Annotated cannot be instantiated', + TypeError, 'Cannot instantiate typing.Annotated', ): Annotated() diff --git a/Lib/typing.py b/Lib/typing.py index 387b4c5ad5284b..8655b756a9fd13 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2001,7 +2001,8 @@ def __mro_entries__(self, bases): return (self.__origin__,) -class Annotated: +@_SpecialForm +def Annotated(self, params): """Add context-specific metadata to a type. Example: Annotated[int, runtime_check.Unsigned] indicates to the @@ -2048,30 +2049,17 @@ class Annotated: where T1, T2 etc. are TypeVars, which would be invalid, because only one type should be passed to Annotated. """ - - __slots__ = () - - def __new__(cls, *args, **kwargs): - raise TypeError("Type Annotated cannot be instantiated.") - - @_tp_cache - def __class_getitem__(cls, params): - if not isinstance(params, tuple) or len(params) < 2: - raise TypeError("Annotated[...] should be used " - "with at least two arguments (a type and an " - "annotation).") - if _is_unpacked_typevartuple(params[0]): - raise TypeError("Annotated[...] should not be used with an " - "unpacked TypeVarTuple") - msg = "Annotated[t, ...]: t must be a type." - origin = _type_check(params[0], msg, allow_special_forms=True) - metadata = tuple(params[1:]) - return _AnnotatedAlias(origin, metadata) - - def __init_subclass__(cls, *args, **kwargs): - raise TypeError( - "Cannot subclass {}.Annotated".format(cls.__module__) - ) + if not isinstance(params, tuple) or len(params) < 2: + raise TypeError("Annotated[...] should be used " + "with at least two arguments (a type and an " + "annotation).") + if _is_unpacked_typevartuple(params[0]): + raise TypeError("Annotated[...] should not be used with an " + "unpacked TypeVarTuple") + msg = "Annotated[t, ...]: t must be a type." + origin = _type_check(params[0], msg, allow_special_forms=True) + metadata = tuple(params[1:]) + return _AnnotatedAlias(origin, metadata) def runtime_checkable(cls): diff --git a/Misc/NEWS.d/next/Library/2023-06-08-15-56-45.gh-issue-105509.YIG57j.rst b/Misc/NEWS.d/next/Library/2023-06-08-15-56-45.gh-issue-105509.YIG57j.rst new file mode 100644 index 00000000000000..26d8a66c2c4e03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-08-15-56-45.gh-issue-105509.YIG57j.rst @@ -0,0 +1,3 @@ +:data:`typing.Annotated` is now implemented as an instance of +``typing._SpecialForm`` rather than a class. This should have no user-facing +impact for users of the :mod:`typing` module public API. From 3b73f9f00e44d6b709fa27f83d64d7355e1891ca Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 2 Sep 2023 00:04:07 +0300 Subject: [PATCH 449/598] gh-107862: Add roundtrip `hypothesis` tests to `test_binascii` (#107863) --- Lib/test/test_binascii.py | 52 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index a2d7d0293ce1ae..eb831b1a06fcb8 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -5,6 +5,7 @@ import array import re from test.support import bigmemtest, _1G, _4G +from test.support.hypothesis_helper import hypothesis # Note: "*_hex" functions are aliases for "(un)hexlify" @@ -27,6 +28,14 @@ class BinASCIITest(unittest.TestCase): def setUp(self): self.data = self.type2test(self.rawdata) + def assertConversion(self, original, converted, restored, **kwargs): + self.assertIsInstance(original, bytes) + self.assertIsInstance(converted, bytes) + self.assertIsInstance(restored, bytes) + if converted: + self.assertLess(max(converted), 128) + self.assertEqual(original, restored, msg=f'{self.type2test=} {kwargs=}') + def test_exceptions(self): # Check module exceptions self.assertTrue(issubclass(binascii.Error, Exception)) @@ -52,9 +61,7 @@ def test_returned_value(self): self.fail("{}/{} conversion raises {!r}".format(fb, fa, err)) self.assertEqual(res, raw, "{}/{} conversion: " "{!r} != {!r}".format(fb, fa, res, raw)) - self.assertIsInstance(res, bytes) - self.assertIsInstance(a, bytes) - self.assertLess(max(a), 128) + self.assertConversion(raw, a, res) self.assertIsInstance(binascii.crc_hqx(raw, 0), int) self.assertIsInstance(binascii.crc32(raw), int) @@ -222,6 +229,15 @@ def test_uu(self): with self.assertRaises(TypeError): binascii.b2a_uu(b"", True) + @hypothesis.given( + binary=hypothesis.strategies.binary(), + backtick=hypothesis.strategies.booleans(), + ) + def test_hex_roundtrip(self, binary, backtick): + converted = binascii.b2a_uu(self.type2test(binary), backtick=backtick) + restored = binascii.a2b_uu(self.type2test(converted)) + self.assertConversion(binary, converted, restored, backtick=backtick) + def test_crc_hqx(self): crc = binascii.crc_hqx(self.type2test(b"Test the CRC-32 of"), 0) crc = binascii.crc_hqx(self.type2test(b" this string."), crc) @@ -259,6 +275,12 @@ def test_hex(self): self.assertEqual(binascii.hexlify(self.type2test(s)), t) self.assertEqual(binascii.unhexlify(self.type2test(t)), u) + @hypothesis.given(binary=hypothesis.strategies.binary()) + def test_hex_roundtrip(self, binary): + converted = binascii.hexlify(self.type2test(binary)) + restored = binascii.unhexlify(self.type2test(converted)) + self.assertConversion(binary, converted, restored) + def test_hex_separator(self): """Test that hexlify and b2a_hex are binary versions of bytes.hex.""" # Logic of separators is tested in test_bytes.py. This checks that @@ -373,6 +395,21 @@ def test_qp(self): self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n') self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E') + @hypothesis.given( + binary=hypothesis.strategies.binary(), + quotetabs=hypothesis.strategies.booleans(), + istext=hypothesis.strategies.booleans(), + header=hypothesis.strategies.booleans(), + ) + def test_b2a_qp_a2b_qp_round_trip(self, binary, quotetabs, istext, header): + converted = binascii.b2a_qp( + self.type2test(binary), + quotetabs=quotetabs, istext=istext, header=header, + ) + restored = binascii.a2b_qp(self.type2test(converted), header=header) + self.assertConversion(binary, converted, restored, + quotetabs=quotetabs, istext=istext, header=header) + def test_empty_string(self): # A test for SF bug #1022953. Make sure SystemError is not raised. empty = self.type2test(b'') @@ -428,6 +465,15 @@ def test_b2a_base64_newline(self): self.assertEqual(binascii.b2a_base64(b, newline=False), b'aGVsbG8=') + @hypothesis.given( + binary=hypothesis.strategies.binary(), + newline=hypothesis.strategies.booleans(), + ) + def test_base64_roundtrip(self, binary, newline): + converted = binascii.b2a_base64(self.type2test(binary), newline=newline) + restored = binascii.a2b_base64(self.type2test(converted)) + self.assertConversion(binary, converted, restored, newline=newline) + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): From d5c5d4bfd3260219397326795d3b2ff62a9ab8cb Mon Sep 17 00:00:00 2001 From: William Andrea Date: Fri, 1 Sep 2023 18:31:21 -0400 Subject: [PATCH 450/598] gh-106392: Fix inconsistency in deprecation warnings (#106436) They used "datetime" to refer to both the object and the module. --- Lib/_pydatetime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index f4fc2c58e5e293..549fcda19dccf2 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1812,7 +1812,7 @@ def utcfromtimestamp(cls, t): warnings.warn("datetime.utcfromtimestamp() is deprecated and scheduled " "for removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " - "datetime.fromtimestamp(t, datetime.UTC).", + "datetime.datetime.fromtimestamp(t, datetime.UTC).", DeprecationWarning, stacklevel=2) return cls._fromtimestamp(t, True, None) @@ -1830,7 +1830,7 @@ def utcnow(cls): warnings.warn("datetime.utcnow() is deprecated and scheduled for " "removal in a future version. Instead, Use timezone-aware " "objects to represent datetimes in UTC: " - "datetime.now(datetime.UTC).", + "datetime.datetime.now(datetime.UTC).", DeprecationWarning, stacklevel=2) t = _time.time() From 76ce537fb1291f90c6dccbfea8653544de7902fd Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 1 Sep 2023 16:27:09 -0700 Subject: [PATCH 451/598] Fix test_regrtest when run with uops always on (#108778) The fix has two parts: - When `-X uops` is detected, pass it on to the subprocess created to run the manufactured test. I need this so I can run `./python -Xuops -m test test_regrtest` and see it fail without the next fix. - Use `-R 6:3:` in `ArgsTestCase.test_huntrleaks` instead of `-R 3:3:` -- it takes longer to settle with `-X uops`. --- Lib/test/test_regrtest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 806b932a164df8..0c1400c8105037 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -548,7 +548,11 @@ def run_command(self, args, input=None, exitcode=0, **kw): return proc def run_python(self, args, **kw): - args = [sys.executable, '-X', 'faulthandler', '-I', *args] + extraargs = [] + if 'uops' in sys._xoptions: + # Pass -X uops along + extraargs.extend(['-X', 'uops']) + args = [sys.executable, *extraargs, '-X', 'faulthandler', '-I', *args] proc = self.run_command(args, **kw) return proc.stdout @@ -893,12 +897,12 @@ def check_leak(self, code, what): filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) - output = self.run_tests('--huntrleaks', '3:3:', test, + output = self.run_tests('--huntrleaks', '6:3:', test, exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test) - line = 'beginning 6 repetitions\n123456\n......\n' + line = 'beginning 9 repetitions\n123456789\n.........\n' self.check_line(output, re.escape(line)) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) From aa52888e6a0269f0c31a24bd0d1adb3238147261 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sat, 2 Sep 2023 00:46:36 +0000 Subject: [PATCH 452/598] gh-108777: Split _PyTime tests from _testinternalcapi.c (gh-108787) --- Makefile.pre.in | 1 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/pytime.c | 0 Modules/_testinternalcapi.c | 267 +-------------------- Modules/_testinternalcapi/README.txt | 5 + Modules/_testinternalcapi/parts.h | 15 ++ Modules/_testinternalcapi/pytime.c | 279 ++++++++++++++++++++++ PCbuild/_testinternalcapi.vcxproj | 1 + PCbuild/_testinternalcapi.vcxproj.filters | 3 + configure | 1 + configure.ac | 1 + 11 files changed, 313 insertions(+), 262 deletions(-) delete mode 100644 Modules/_testcapi/pytime.c create mode 100644 Modules/_testinternalcapi/README.txt create mode 100644 Modules/_testinternalcapi/parts.h create mode 100644 Modules/_testinternalcapi/pytime.c diff --git a/Makefile.pre.in b/Makefile.pre.in index 5826b9a82736b9..7b67738f4341a2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2800,6 +2800,7 @@ MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/H MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 2bb39d2936894a..56c1badf6b44a0 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -158,7 +158,7 @@ @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c -@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c +@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/pytime.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/pytime.c b/Modules/_testcapi/pytime.c deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e375ca8556d217..96d568294a9710 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -33,9 +33,8 @@ #include "clinic/_testinternalcapi.c.h" -#ifdef MS_WINDOWS -# include // struct timeval -#endif +// Include test definitions from _testinternalcapi/ +#include "_testinternalcapi/parts.h" #define MODULE_NAME "_testinternalcapi" @@ -1123,250 +1122,6 @@ pending_identify(PyObject *self, PyObject *args) return res; } - -static PyObject * -test_pytime_fromseconds(PyObject *self, PyObject *args) -{ - int seconds; - if (!PyArg_ParseTuple(args, "i", &seconds)) { - return NULL; - } - _PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsNanosecondsObject(ts); -} - -static int -check_time_rounding(int round) -{ - if (round != _PyTime_ROUND_FLOOR - && round != _PyTime_ROUND_CEILING - && round != _PyTime_ROUND_HALF_EVEN - && round != _PyTime_ROUND_UP) - { - PyErr_SetString(PyExc_ValueError, "invalid rounding"); - return -1; - } - return 0; -} - -static PyObject * -test_pytime_fromsecondsobject(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { - return NULL; - } - return _PyTime_AsNanosecondsObject(ts); -} - -static PyObject * -test_pytime_assecondsdouble(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { - return NULL; - } - double d = _PyTime_AsSecondsDouble(ts); - return PyFloat_FromDouble(d); -} - -static PyObject * -test_PyTime_AsTimeval(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timeval tv; - if (_PyTime_AsTimeval(t, &tv, round) < 0) { - return NULL; - } - - PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); - if (seconds == NULL) { - return NULL; - } - return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); -} - -static PyObject * -test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timeval tv; - _PyTime_AsTimeval_clamp(t, &tv, round); - - PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); - if (seconds == NULL) { - return NULL; - } - return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); -} - -#ifdef HAVE_CLOCK_GETTIME -static PyObject * -test_PyTime_AsTimespec(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timespec ts; - if (_PyTime_AsTimespec(t, &ts) == -1) { - return NULL; - } - return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); -} - -static PyObject * -test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timespec ts; - _PyTime_AsTimespec_clamp(t, &ts); - return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); -} -#endif - -static PyObject * -test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t ms = _PyTime_AsMilliseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(ms); - return _PyTime_AsNanosecondsObject(ns); -} - -static PyObject * -test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t us = _PyTime_AsMicroseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(us); - return _PyTime_AsNanosecondsObject(ns); -} - -static PyObject * -test_pytime_object_to_time_t(PyObject *self, PyObject *args) -{ - PyObject *obj; - time_t sec; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - if (_PyTime_ObjectToTime_t(obj, &sec, round) == -1) { - return NULL; - } - return _PyLong_FromTime_t(sec); -} - -static PyObject * -test_pytime_object_to_timeval(PyObject *self, PyObject *args) -{ - PyObject *obj; - time_t sec; - long usec; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - if (_PyTime_ObjectToTimeval(obj, &sec, &usec, round) == -1) { - return NULL; - } - return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), usec); -} - -static PyObject * -test_pytime_object_to_timespec(PyObject *self, PyObject *args) -{ - PyObject *obj; - time_t sec; - long nsec; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - if (_PyTime_ObjectToTimespec(obj, &sec, &nsec, round) == -1) { - return NULL; - } - return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); -} - - static PyObject * tracemalloc_get_traceback(PyObject *self, PyObject *args) { @@ -1731,20 +1486,6 @@ static PyMethodDef module_functions[] = { {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, - {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, - {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, - {"_PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, -#ifdef HAVE_CLOCK_GETTIME - {"_PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, - {"_PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, -#endif - {"_PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS}, - {"_PyTime_AsTimeval_clamp", test_PyTime_AsTimeval_clamp, METH_VARARGS}, - {"_PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS}, - {"_PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, - {"_PyTime_ObjectToTime_t", test_pytime_object_to_time_t, METH_VARARGS}, - {"_PyTime_ObjectToTimespec", test_pytime_object_to_timespec, METH_VARARGS}, - {"_PyTime_ObjectToTimeval", test_pytime_object_to_timeval, METH_VARARGS}, {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS}, {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O}, @@ -1771,6 +1512,10 @@ static PyMethodDef module_functions[] = { static int module_exec(PyObject *module) { + if (_PyTestInternalCapi_Init_PyTime(module) < 0) { + return 1; + } + if (PyModule_Add(module, "SIZEOF_PYGC_HEAD", PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { return 1; diff --git a/Modules/_testinternalcapi/README.txt b/Modules/_testinternalcapi/README.txt new file mode 100644 index 00000000000000..eb071135e4c592 --- /dev/null +++ b/Modules/_testinternalcapi/README.txt @@ -0,0 +1,5 @@ +Tests in this directory are compiled into the _testinternalcapi extension. +The main file for the extension is Modules/_testinternalcapimodule.c, which +calls `_PyTestInternalCapi_Init_*` from these functions. + +See Modules/_testcapi/README.txt for guideline when writing C test code. \ No newline at end of file diff --git a/Modules/_testinternalcapi/parts.h b/Modules/_testinternalcapi/parts.h new file mode 100644 index 00000000000000..43e7714b235156 --- /dev/null +++ b/Modules/_testinternalcapi/parts.h @@ -0,0 +1,15 @@ +#ifndef Py_TESTINTERNALCAPI_PARTS_H +#define Py_TESTINTERNALCAPI_PARTS_H + +// Always enable assertions +#undef NDEBUG + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" + +int _PyTestInternalCapi_Init_PyTime(PyObject *module); + +#endif // Py_TESTINTERNALCAPI_PARTS_H diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c new file mode 100644 index 00000000000000..2b5f9eb0ef2851 --- /dev/null +++ b/Modules/_testinternalcapi/pytime.c @@ -0,0 +1,279 @@ +/* Test pycore_time.h */ + +#include "parts.h" + +#include "pycore_time.h" + +#ifdef MS_WINDOWS +# include // struct timeval +#endif + + +static PyObject * +test_pytime_fromseconds(PyObject *self, PyObject *args) +{ + int seconds; + if (!PyArg_ParseTuple(args, "i", &seconds)) { + return NULL; + } + _PyTime_t ts = _PyTime_FromSeconds(seconds); + return _PyTime_AsNanosecondsObject(ts); +} + +static int +check_time_rounding(int round) +{ + if (round != _PyTime_ROUND_FLOOR + && round != _PyTime_ROUND_CEILING + && round != _PyTime_ROUND_HALF_EVEN + && round != _PyTime_ROUND_UP) + { + PyErr_SetString(PyExc_ValueError, "invalid rounding"); + return -1; + } + return 0; +} + +static PyObject * +test_pytime_fromsecondsobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t ts; + if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { + return NULL; + } + return _PyTime_AsNanosecondsObject(ts); +} + +static PyObject * +test_pytime_assecondsdouble(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t ts; + if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { + return NULL; + } + double d = _PyTime_AsSecondsDouble(ts); + return PyFloat_FromDouble(d); +} + +static PyObject * +test_PyTime_AsTimeval(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timeval tv; + if (_PyTime_AsTimeval(t, &tv, round) < 0) { + return NULL; + } + + PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); + if (seconds == NULL) { + return NULL; + } + return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); +} + +static PyObject * +test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timeval tv; + _PyTime_AsTimeval_clamp(t, &tv, round); + + PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); + if (seconds == NULL) { + return NULL; + } + return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); +} + +#ifdef HAVE_CLOCK_GETTIME +static PyObject * +test_PyTime_AsTimespec(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timespec ts; + if (_PyTime_AsTimespec(t, &ts) == -1) { + return NULL; + } + return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); +} + +static PyObject * +test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timespec ts; + _PyTime_AsTimespec_clamp(t, &ts); + return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); +} +#endif + +static PyObject * +test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t ms = _PyTime_AsMilliseconds(t, round); + _PyTime_t ns = _PyTime_FromNanoseconds(ms); + return _PyTime_AsNanosecondsObject(ns); +} + +static PyObject * +test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t us = _PyTime_AsMicroseconds(t, round); + _PyTime_t ns = _PyTime_FromNanoseconds(us); + return _PyTime_AsNanosecondsObject(ns); +} + +static PyObject * +test_pytime_object_to_time_t(PyObject *self, PyObject *args) +{ + PyObject *obj; + time_t sec; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + if (_PyTime_ObjectToTime_t(obj, &sec, round) == -1) { + return NULL; + } + return _PyLong_FromTime_t(sec); +} + +static PyObject * +test_pytime_object_to_timeval(PyObject *self, PyObject *args) +{ + PyObject *obj; + time_t sec; + long usec; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + if (_PyTime_ObjectToTimeval(obj, &sec, &usec, round) == -1) { + return NULL; + } + return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), usec); +} + +static PyObject * +test_pytime_object_to_timespec(PyObject *self, PyObject *args) +{ + PyObject *obj; + time_t sec; + long nsec; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + if (_PyTime_ObjectToTimespec(obj, &sec, &nsec, round) == -1) { + return NULL; + } + return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); +} + +static PyMethodDef TestMethods[] = { + {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, + {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, + {"_PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, +#ifdef HAVE_CLOCK_GETTIME + {"_PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, + {"_PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, +#endif + {"_PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS}, + {"_PyTime_AsTimeval_clamp", test_PyTime_AsTimeval_clamp, METH_VARARGS}, + {"_PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS}, + {"_PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, + {"_PyTime_ObjectToTime_t", test_pytime_object_to_time_t, METH_VARARGS}, + {"_PyTime_ObjectToTimespec", test_pytime_object_to_timespec, METH_VARARGS}, + {"_PyTime_ObjectToTimeval", test_pytime_object_to_timeval, METH_VARARGS}, + {NULL, NULL} /* sentinel */ +}; + +int +_PyTestInternalCapi_Init_PyTime(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index 6c5b12cd40e983..59491c644b6655 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -94,6 +94,7 @@ + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index 7734da0b7b426b..21a66a2aa79f76 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -12,6 +12,9 @@ Source Files + + Source Files + diff --git a/configure b/configure index 7fe4aead29a732..983b1edf4a86e1 100755 --- a/configure +++ b/configure @@ -26553,6 +26553,7 @@ SRCDIRS="\ Modules/_sqlite \ Modules/_sre \ Modules/_testcapi \ + Modules/_testinternalcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ diff --git a/configure.ac b/configure.ac index 5673b374353b47..10f132faa9d80d 100644 --- a/configure.ac +++ b/configure.ac @@ -6470,6 +6470,7 @@ SRCDIRS="\ Modules/_sqlite \ Modules/_sre \ Modules/_testcapi \ + Modules/_testinternalcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ From f3ba0a74cd50274acdcd592d4ce8395b92492b7c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 2 Sep 2023 07:45:34 +0300 Subject: [PATCH 453/598] gh-108416: Mark slow test methods with @requires_resource('cpu') (GH-108421) Only mark tests which spend significant system or user time, by itself or in subprocesses. --- Lib/test/_test_multiprocessing.py | 2 ++ Lib/test/pickletester.py | 1 + Lib/test/test_ast.py | 1 + Lib/test/test_buffer.py | 2 ++ Lib/test/test_builtin.py | 1 + Lib/test/test_compile.py | 1 + Lib/test/test_compileall.py | 2 ++ Lib/test/test_concurrent_futures/test_thread_pool.py | 2 ++ Lib/test/test_context.py | 2 ++ Lib/test/test_cppext/__init__.py | 2 ++ Lib/test/test_descr.py | 1 + Lib/test/test_email/test_email.py | 2 ++ Lib/test/test_exceptions.py | 1 + Lib/test/test_gdb.py | 7 +++++++ Lib/test/test_interpreters.py | 2 ++ Lib/test/test_itertools.py | 1 + Lib/test/test_largefile.py | 4 +++- Lib/test/test_multibytecodec.py | 1 + Lib/test/test_peepholer.py | 2 ++ Lib/test/test_runpy.py | 3 ++- Lib/test/test_selectors.py | 1 + Lib/test/test_source_encoding.py | 3 ++- Lib/test/test_statistics.py | 2 ++ Lib/test/test_subprocess.py | 1 + Lib/test/test_support.py | 1 + Lib/test/test_sys_settrace.py | 1 + Lib/test/test_tools/test_freeze.py | 4 +--- Lib/test/test_trace.py | 3 ++- Lib/test/test_traceback.py | 1 + Lib/test/test_unicodedata.py | 2 ++ Lib/test/test_venv.py | 4 +++- Lib/test/test_weakref.py | 2 ++ 32 files changed, 57 insertions(+), 8 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 7a851d3df78781..bcbb4c2929d69e 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -329,6 +329,7 @@ def test_set_executable(self): p.join() self.assertEqual(p.exitcode, 0) + @support.requires_resource('cpu') def test_args_argument(self): # bpo-45735: Using list or tuple as *args* in constructor could # achieve the same effect. @@ -4504,6 +4505,7 @@ def test_finalize(self): result = [obj for obj in iter(conn.recv, 'STOP')] self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e']) + @support.requires_resource('cpu') def test_thread_safety(self): # bpo-24484: _run_finalizers() should be thread-safe def cb(): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 6e87370c2065ba..a687fe0629080a 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2576,6 +2576,7 @@ def check_frame_opcodes(self, pickled): self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_framing_many_objects(self): obj = list(range(10**5)) for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 68de4d6f3f1923..fa96e259b8c978 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -2031,6 +2031,7 @@ def test_nameconstant(self): 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', ]) + @support.requires_resource('cpu') def test_stdlib_validates(self): stdlib = os.path.dirname(ast.__file__) tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index b679d2e8a01f92..72a06d6af450e3 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -1029,6 +1029,7 @@ def match(req, flag): ndim=ndim, shape=shape, strides=strides, lst=lst, sliced=sliced) + @support.requires_resource('cpu') def test_ndarray_getbuf(self): requests = ( # distinct flags @@ -2760,6 +2761,7 @@ def iter_roundtrip(ex, m, items, fmt): m = memoryview(ex) iter_roundtrip(ex, m, items, fmt) + @support.requires_resource('cpu') def test_memoryview_cast_1D_ND(self): # Cast between C-contiguous buffers. At least one buffer must # be 1D, at least one format must be 'c', 'b' or 'B'. diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index dbc706ae7b41f5..33cb248ff6e82b 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -952,6 +952,7 @@ def test_filter_pickle(self): f2 = filter(filter_char, "abcdeabcde") self.check_iter_pickle(f1, list(f2), proto) + @support.requires_resource('cpu') def test_filter_dealloc(self): # Tests recursive deallocation of nested filter objects using the # thrashcan mechanism. See gh-102356 for more details. diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index a470d6187aa673..de513daf825d81 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -784,6 +784,7 @@ def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + @support.requires_resource('cpu') def test_stack_overflow(self): # bpo-31113: Stack overflow when compile a long sequence of # complex statements. diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 05154c8f1c6057..df7c5122b3b1f5 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -551,6 +551,7 @@ def test_no_args_compiles_path(self): self.assertNotCompiled(self.barfn) @without_source_date_epoch # timestamp invalidation test + @support.requires_resource('cpu') def test_no_args_respects_force_flag(self): bazfn = script_helper.make_script(self.directory, 'baz', '') with self.temporary_pycache_prefix() as env: @@ -568,6 +569,7 @@ def test_no_args_respects_force_flag(self): mtime2 = os.stat(pycpath).st_mtime self.assertNotEqual(mtime, mtime2) + @support.requires_resource('cpu') def test_no_args_respects_quiet_flag(self): script_helper.make_script(self.directory, 'baz', '') with self.temporary_pycache_prefix() as env: diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index daef7b5a836825..812f989d8f3ad2 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -6,6 +6,7 @@ import threading import unittest from concurrent import futures +from test import support from .executor import ExecutorTest, mul from .util import BaseTestCase, ThreadPoolMixin, setup_module @@ -49,6 +50,7 @@ def test_idle_thread_reuse(self): executor.shutdown(wait=True) @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + @support.requires_resource('cpu') def test_hang_global_shutdown_lock(self): # bpo-45021: _global_shutdown_lock should be reinitialized in the child # process, otherwise it will never exit diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index b72d5addc6bd7d..255be306156c0b 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -6,6 +6,7 @@ import time import unittest import weakref +from test import support from test.support import threading_helper try: @@ -570,6 +571,7 @@ def test_hamt_collision_3(self): self.assertEqual({k.name for k in h.keys()}, {'C', 'D', 'E'}) + @support.requires_resource('cpu') def test_hamt_stress(self): COLLECTION_SIZE = 7000 TEST_ITERS_EVERY = 647 diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 8adff91c38efcf..74bf420900367e 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -16,9 +16,11 @@ @support.requires_subprocess() class TestCPPExt(unittest.TestCase): + @support.requires_resource('cpu') def test_build_cpp11(self): self.check_build(False, '_testcpp11ext') + @support.requires_resource('cpu') def test_build_cpp03(self): self.check_build(True, '_testcpp03ext') diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index fc8d0a305d8da0..f6bd9094e8fece 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4457,6 +4457,7 @@ class Oops(object): o.whatever = Provoker(o) del o + @support.requires_resource('cpu') def test_wrapper_segfault(self): # SF 927248: deeply nested wrappers could cause stack overflow f = lambda:None diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index cdb6ef1275e520..2a237095b9080c 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -38,6 +38,7 @@ from email import quoprimime from email import utils +from test import support from test.support import threading_helper from test.support.os_helper import unlink from test.test_email import openfile, TestEmailBase @@ -3358,6 +3359,7 @@ def test_getaddresses_header_obj(self): self.assertEqual(addrs[0][1], 'aperson@dom.ain') @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_make_msgid_collisions(self): # Test make_msgid uniqueness, even with multiple threads class MsgidsThread(Thread): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c766f4d43311eb..106baf959a6898 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1350,6 +1350,7 @@ def g(): @cpython_only + @support.requires_resource('cpu') def test_trashcan_recursion(self): # See bpo-33930 diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 85009089f21d2f..c05a2d387c429c 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -317,6 +317,7 @@ def assertGdbRepr(self, val, exp_repr=None): ('%r did not equal expected %r; full output was:\n%s' % (gdb_repr, exp_repr, gdb_output))) + @support.requires_resource('cpu') def test_int(self): 'Verify the pretty-printing of various int values' self.assertGdbRepr(42) @@ -343,6 +344,7 @@ def test_lists(self): self.assertGdbRepr([]) self.assertGdbRepr(list(range(5))) + @support.requires_resource('cpu') def test_bytes(self): 'Verify the pretty-printing of bytes' self.assertGdbRepr(b'') @@ -357,6 +359,7 @@ def test_bytes(self): self.assertGdbRepr(bytes([b for b in range(255)])) + @support.requires_resource('cpu') def test_strings(self): 'Verify the pretty-printing of unicode strings' # We cannot simply call locale.getpreferredencoding() here, @@ -407,6 +410,7 @@ def test_tuples(self): self.assertGdbRepr((1,), '(1,)') self.assertGdbRepr(('foo', 'bar', 'baz')) + @support.requires_resource('cpu') def test_sets(self): 'Verify the pretty-printing of sets' if (gdb_major_version, gdb_minor_version) < (7, 3): @@ -425,6 +429,7 @@ def test_sets(self): id(s)''') self.assertEqual(gdb_repr, "{'b'}") + @support.requires_resource('cpu') def test_frozensets(self): 'Verify the pretty-printing of frozensets' if (gdb_major_version, gdb_minor_version) < (7, 3): @@ -828,6 +833,7 @@ def test_bt_full(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") + @support.requires_resource('cpu') def test_threads(self): 'Verify that "py-bt" indicates threads that are waiting for the GIL' cmd = ''' @@ -889,6 +895,7 @@ def test_gc(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") + @support.requires_resource('cpu') # Some older versions of gdb will fail with # "Cannot find new threads: generic error" # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 5981d96de8de06..90932c0f66f38f 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -469,12 +469,14 @@ class StressTests(TestBase): # In these tests we generally want a lot of interpreters, # but not so many that any test takes too long. + @support.requires_resource('cpu') def test_create_many_sequential(self): alive = [] for _ in range(100): interp = interpreters.create() alive.append(interp) + @support.requires_resource('cpu') def test_create_many_threaded(self): alive = [] def task(): diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 4d6ea780e15373..512745e45350d1 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -2401,6 +2401,7 @@ def gen2(x): self.assertEqual(hist, [0,1]) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_long_chain_of_empty_iterables(self): # Make sure itertools.chain doesn't run into recursion limits when # dealing with long chains of empty iterables. Even with a high diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 3c11c59baef6e5..3b0930fe69e30e 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -8,7 +8,7 @@ import socket import shutil import threading -from test.support import requires, bigmemtest +from test.support import requires, bigmemtest, requires_resource from test.support import SHORT_TIMEOUT from test.support import socket_helper from test.support.os_helper import TESTFN, unlink @@ -173,6 +173,7 @@ class TestCopyfile(LargeFileTest, unittest.TestCase): # Exact required disk space would be (size * 2), but let's give it a # bit more tolerance. @skip_no_disk_space(TESTFN, size * 2.5) + @requires_resource('cpu') def test_it(self): # Internally shutil.copyfile() can use "fast copy" methods like # os.sendfile(). @@ -222,6 +223,7 @@ def run(sock): # Exact required disk space would be (size * 2), but let's give it a # bit more tolerance. @skip_no_disk_space(TESTFN, size * 2.5) + @requires_resource('cpu') def test_it(self): port = socket_helper.find_unused_port() with socket.create_server(("", port)) as sock: diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index cf8bb5e3a0520d..6451df14696933 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -363,6 +363,7 @@ def test_iso2022_jp_g0(self): e = '\u3406'.encode(encoding) self.assertFalse(any(x > 0x80 for x in e)) + @support.requires_resource('cpu') def test_bug1572832(self): for x in range(0x10000, 0x110000): # Any ISO 2022 codec will cause the segfault diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index fba41f0b119796..b825b27060e86b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -4,6 +4,7 @@ import textwrap import unittest +from test import support from test.support.bytecode_helper import BytecodeTestCase, CfgOptimizationTestCase @@ -522,6 +523,7 @@ def genexpr(): return (y for x in a for y in [f(x)]) self.assertEqual(count_instr_recursively(genexpr, 'FOR_ITER'), 1) + @support.requires_resource('cpu') def test_format_combinations(self): flags = '-+ #0' testcases = [ diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 6aaa288c14e1d7..628c8cae38a751 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,7 +12,7 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose, requires_subprocess +from test.support import no_tracing, verbose, requires_subprocess, requires_resource from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir from test.support.script_helper import make_script, make_zip_script @@ -733,6 +733,7 @@ def test_zipfile_error(self): self._check_import_error(zip_name, msg) @no_tracing + @requires_resource('cpu') def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 4545cbadb796fd..33417ca6a11af0 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -455,6 +455,7 @@ class ScalableSelectorMixIn: # see issue #18963 for why it's skipped on older OS X versions @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") + @support.requires_resource('cpu') def test_above_fd_setsize(self): # A scalable implementation should have no problem with more than # FD_SETSIZE file descriptors. Since we don't know the value, we just diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index b05173ad00d442..72c2b47779e005 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import unittest -from test.support import script_helper, captured_stdout, requires_subprocess +from test.support import script_helper, captured_stdout, requires_subprocess, requires_resource from test.support.os_helper import TESTFN, unlink, rmtree from test.support.import_helper import unload import importlib @@ -250,6 +250,7 @@ def test_crcrcrlf2(self): class UTF8ValidatorTest(unittest.TestCase): @unittest.skipIf(not sys.platform.startswith("linux"), "Too slow to run on non-Linux platforms") + @requires_resource('cpu') def test_invalid_utf8(self): # This is a port of test_utf8_decode_invalid_sequences in # test_unicode.py to exercise the separate utf8 validator in diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index aa2cf2b1edc584..23a3973305303d 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2145,6 +2145,7 @@ def test_integer_sqrt_of_frac_rto(self): self.assertTrue(m * (r - 1)**2 < n < m * (r + 1)**2) @requires_IEEE_754 + @support.requires_resource('cpu') def test_float_sqrt_of_frac(self): def is_root_correctly_rounded(x: Fraction, root: float) -> bool: @@ -2849,6 +2850,7 @@ def test_cdf(self): self.assertTrue(math.isnan(X.cdf(float('NaN')))) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_inv_cdf(self): NormalDist = self.module.NormalDist diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 817eab0c8a7e19..0b9e9e16f55d7e 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1292,6 +1292,7 @@ def test_bufsize_equal_one_binary_mode(self): with self.assertWarnsRegex(RuntimeWarning, 'line buffering'): self._test_bufsize_equal_one(line, b'', universal_newlines=False) + @support.requires_resource('cpu') def test_leaking_fds_on_error(self): # see bug #5179: Popen leaks file descriptors to PIPEs if # the child fails to execute; this will eventually exhaust diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index cd1679dd23281c..86d26b7e8df4d0 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -497,6 +497,7 @@ def check_options(self, args, func, expected=None): self.assertEqual(proc.stdout.rstrip(), repr(expected)) self.assertEqual(proc.returncode, 0) + @support.requires_resource('cpu') def test_args_from_interpreter_flags(self): # Test test.support.args_from_interpreter_flags() for opts in ( diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 4462b5c712d662..7d38addaee413e 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2834,6 +2834,7 @@ def test_jump_extended_args_unpack_ex_tricky(output): ) = output.append(4) or "Spam" output.append(5) + @support.requires_resource('cpu') def test_jump_extended_args_for_iter(self): # In addition to failing when extended arg handling is broken, this can # also hang for a *very* long time: diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 3e9a48b5bc6a89..922e74b441457a 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -17,10 +17,8 @@ @support.skip_if_buildbot('not all buildbots have enough space') class TestFreeze(unittest.TestCase): + @support.requires_resource('cpu') # Building Python is slow def test_freeze_simple_script(self): - # Building Python is slow - support.requires('cpu') - script = textwrap.dedent(""" import sys print('running...') diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 73339ebdb7c4e9..d1ef005a4314ed 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,7 +1,7 @@ import os from pickle import dump import sys -from test.support import captured_stdout +from test.support import captured_stdout, requires_resource from test.support.os_helper import (TESTFN, rmtree, unlink) from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap @@ -367,6 +367,7 @@ def _coverage(self, tracer, r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=TESTFN) + @requires_resource('cpu') def test_coverage(self): tracer = trace.Trace(trace=0, count=1) with captured_stdout() as stdout: diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 7c6fdbf762921f..316ade2171e94f 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3539,6 +3539,7 @@ def CHECK(a, b, expected): CHECK("AttributeError", "AttributeErrorTests", 10) CHECK("ABA", "AAB", 4) + @support.requires_resource('cpu') def test_levenshtein_distance_short_circuit(self): if not LEVENSHTEIN_DATA_FILE.is_file(): self.fail( diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 3dc0790ca15b41..515c3840cb3647 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -313,6 +313,7 @@ def test_ucd_510(self): self.assertTrue("\u1d79".upper()=='\ua77d') self.assertTrue(".".upper()=='.') + @requires_resource('cpu') def test_bug_5828(self): self.assertEqual("\u1d79".lower(), "\u1d79") # Only U+0000 should have U+0000 as its upper/lower/titlecase variant @@ -353,6 +354,7 @@ def unistr(data): return "".join([chr(x) for x in data]) @requires_resource('network') + @requires_resource('cpu') def test_normalization(self): TESTDATAFILE = "NormalizationTest.txt" TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}" diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 5205604c2c7185..aa6a8fbf8cfd17 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -20,7 +20,8 @@ from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_emscripten, is_wasi, - requires_venv_with_pip, TEST_HOME_DIR) + requires_venv_with_pip, TEST_HOME_DIR, + requires_resource) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -775,6 +776,7 @@ def nicer_error(self): ) @requires_venv_with_pip() + @requires_resource('cpu') def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 1bc1d05f7daba9..4cdd66d3769e0c 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1933,6 +1933,7 @@ def test_threaded_weak_key_dict_copy(self): self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False) @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_deepcopy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. @@ -1945,6 +1946,7 @@ def test_threaded_weak_value_dict_copy(self): self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False) @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_deepcopy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. From 21da4980f5916e8fd648f04367a9e60d141af366 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 2 Sep 2023 12:32:19 +0300 Subject: [PATCH 454/598] gh-101100: Fix sphinx warnings in `uuid.rst` (#108805) * gh-101100: Fix sphinx warnings in `uuid.rst` * Use anchors --- Doc/library/uuid.rst | 51 ++++++++++++++++++++++++++------------------ Doc/tools/.nitignore | 1 - 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 94b9a432372195..b78e74b4e04f1c 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -22,7 +22,7 @@ random UUID. Depending on support from the underlying platform, :func:`uuid1` may or may not return a "safe" UUID. A safe UUID is one which is generated using synchronization methods that ensure no two processes can obtain the same -UUID. All instances of :class:`UUID` have an :attr:`is_safe` attribute +UUID. All instances of :class:`UUID` have an :attr:`~UUID.is_safe` attribute which relays any information about the UUID's safety, using this enumeration: .. class:: SafeUUID @@ -95,25 +95,34 @@ which relays any information about the UUID's safety, using this enumeration: A tuple of the six integer fields of the UUID, which are also available as six individual attributes and two derived attributes: - +------------------------------+-------------------------------+ - | Field | Meaning | - +==============================+===============================+ - | :attr:`time_low` | the first 32 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time_mid` | the next 16 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time_hi_version` | the next 16 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`clock_seq_hi_variant` | the next 8 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`clock_seq_low` | the next 8 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`node` | the last 48 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time` | the 60-bit timestamp | - +------------------------------+-------------------------------+ - | :attr:`clock_seq` | the 14-bit sequence number | - +------------------------------+-------------------------------+ +.. list-table:: + + * - Field + - Meaning + + * - .. attribute:: UUID.time_low + - The next 32 bits of the UUID. + + * - .. attribute:: UUID.time_mid + - The next 16 bits of the UUID. + + * - .. attribute:: UUID.time_hi_version + - The next 16 bits of the UUID. + + * - .. attribute:: UUID.clock_seq_hi_variant + - The next 8 bits of the UUID. + + * - .. attribute:: UUID.clock_seq_low + - The next 8 bits of the UUID. + + * - .. attribute:: UUID.node + - The last 48 bits of the UUID. + + * - .. attribute:: UUID.time + - The the 60-bit timestamp. + + * - .. attribute:: UUID.clock_seq + - The 14-bit sequence number. .. attribute:: UUID.hex @@ -233,7 +242,7 @@ The :mod:`uuid` module defines the following namespace identifiers for use with text output format. The :mod:`uuid` module defines the following constants for the possible values -of the :attr:`variant` attribute: +of the :attr:`~UUID.variant` attribute: .. data:: RESERVED_NCS diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1d89c9c683a070..c0fc47cc72029c 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -136,7 +136,6 @@ Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst Doc/library/urllib.request.rst -Doc/library/uuid.rst Doc/library/weakref.rst Doc/library/wsgiref.rst Doc/library/xml.dom.minidom.rst From dd05c2054fbed84baf87a707d3c77e0cc74ed82c Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 2 Sep 2023 13:08:46 +0300 Subject: [PATCH 455/598] Fix typo in `uuid.rst` (#108809) --- Doc/library/uuid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index b78e74b4e04f1c..d2ada7f471a752 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -119,7 +119,7 @@ which relays any information about the UUID's safety, using this enumeration: - The last 48 bits of the UUID. * - .. attribute:: UUID.time - - The the 60-bit timestamp. + - The 60-bit timestamp. * - .. attribute:: UUID.clock_seq - The 14-bit sequence number. From 5141b1ebe07ad54279e0770b4704eaf76f24951d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 2 Sep 2023 15:05:44 +0300 Subject: [PATCH 456/598] gh-101100: Fix sphinx warnings in `unittest.mock-examples.rst` (#108810) --- Doc/library/unittest.mock-examples.rst | 13 +++++++------ Doc/library/uuid.rst | 2 +- Doc/tools/.nitignore | 1 - 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 895b9f9f07671b..744fc9de63cd16 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -579,14 +579,14 @@ Partial mocking In some tests I wanted to mock out a call to :meth:`datetime.date.today` to return a known date, but I didn't want to prevent the code under test from creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`date.today` method. +so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. I found a simple way of doing this that involved effectively wrapping the date class with a mock, but passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to -mock out the ``date`` class in the module under test. The :attr:`side_effect` +mock out the ``date`` class in the module under test. The :attr:`~Mock.side_effect` attribute on the mock date class is then set to a lambda function that returns a real date. When the mock date class is called a real date will be constructed and returned by ``side_effect``. :: @@ -766,8 +766,8 @@ mock has a nice API for making assertions about how your mock objects are used. >>> mock.foo_bar.assert_called_with('baz', spam='eggs') If your mock is only being called once you can use the -:meth:`assert_called_once_with` method that also asserts that the -:attr:`call_count` is one. +:meth:`~Mock.assert_called_once_with` method that also asserts that the +:attr:`~Mock.call_count` is one. >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') >>> mock.foo_bar() @@ -835,7 +835,7 @@ One possibility would be for mock to copy the arguments you pass in. This could then cause problems if you do assertions that rely on object identity for equality. -Here's one solution that uses the :attr:`side_effect` +Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this @@ -971,7 +971,8 @@ We can do this with :class:`MagicMock`, which will behave like a dictionary, and using :data:`~Mock.side_effect` to delegate dictionary access to a real underlying dictionary that is under our control. -When the :meth:`__getitem__` and :meth:`__setitem__` methods of our ``MagicMock`` are called +When the :meth:`~object.__getitem__` and :meth:`~object.__setitem__` methods +of our ``MagicMock`` are called (normal dictionary access) then ``side_effect`` is called with the key (and in the case of ``__setitem__`` the value too). We can also control what is returned. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index d2ada7f471a752..adf01770656754 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -101,7 +101,7 @@ which relays any information about the UUID's safety, using this enumeration: - Meaning * - .. attribute:: UUID.time_low - - The next 32 bits of the UUID. + - The first 32 bits of the UUID. * - .. attribute:: UUID.time_mid - The next 16 bits of the UUID. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index c0fc47cc72029c..a6268048e143db 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -131,7 +131,6 @@ Doc/library/tkinter.ttk.rst Doc/library/traceback.rst Doc/library/tty.rst Doc/library/turtle.rst -Doc/library/unittest.mock-examples.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst From 1f3e797dc0451f48e649bcab2c9fd80224ffdac0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 15:46:43 +0200 Subject: [PATCH 457/598] gh-108765: Remove old prototypes from pyport.h (#108782) Move prototypes of gethostname(), _getpty() and struct termios from pyport.h to the C code using them: posixmodule.c, socketmodule.c and termios.c. Replace "#ifdef SOLARIS" with "#ifdef __sun". --- Include/pyport.h | 26 -------------------------- Modules/posixmodule.c | 7 +++++++ Modules/socketmodule.c | 6 +++++- Modules/termios.c | 13 ++++++++++--- 4 files changed, 22 insertions(+), 30 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 511c3fda1a40a5..67164328d29576 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -417,32 +417,6 @@ extern "C" { # define Py_NO_INLINE #endif -/************************************************************************** -Prototypes that are missing from the standard include files on some systems -(and possibly only some versions of such systems.) - -Please be conservative with adding new ones, document them and enclose them -in platform-specific #ifdefs. -**************************************************************************/ - -#ifdef SOLARIS -/* Unchecked */ -extern int gethostname(char *, int); -#endif - -#ifdef HAVE__GETPTY -#include /* we need to import mode_t */ -extern char * _getpty(int *, int, mode_t, int); -#endif - -/* On QNX 6, struct termio must be declared by including sys/termio.h - if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must - be included before termios.h or it will generate an error. */ -#if defined(HAVE_SYS_TERMIO_H) && !defined(__hpux) -#include -#endif - - /* On 4.4BSD-descendants, ctype functions serves the whole range of * wchar_t character set rather than single byte code points only. * This characteristic can break some operations of string object diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0436571abc9054..761542866d8f96 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -58,6 +58,13 @@ #include // ctermid() #include // system() +// SGI apparently needs this forward declaration +#ifdef HAVE__GETPTY +# include // mode_t + extern char * _getpty(int *, int, mode_t, int); +#endif + + /* * A number of APIs are available on macOS from a certain macOS version. * To support building with a new SDK while deploying to older versions diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e3681853dad095..2f12c9cedbd8a6 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -111,9 +111,13 @@ Local naming conventions: #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +// gethostname() prototype missing from Solaris standard header files +#ifdef __sun +extern int gethostname(char *, int); +#endif #ifdef _Py_MEMORY_SANITIZER -# include +# include #endif /* Socket object documentation */ diff --git a/Modules/termios.c b/Modules/termios.c index 21d3541c177bc3..a0613837ef9795 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -6,10 +6,17 @@ #include "Python.h" -/* Apparently, on SGI, termios.h won't define CTRL if _XOPEN_SOURCE - is defined, so we define it here. */ +// On QNX 6, struct termio must be declared by including sys/termio.h +// if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must +// be included before termios.h or it will generate an error. +#if defined(HAVE_SYS_TERMIO_H) && !defined(__hpux) +# include +#endif + +// Apparently, on SGI, termios.h won't define CTRL if _XOPEN_SOURCE +// is defined, so we define it here. #if defined(__sgi) -#define CTRL(c) ((c)&037) +# define CTRL(c) ((c)&037) #endif #if defined(__sun) From 4ba18099b70c9f20f69357bac94d74f7c3238d7f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 15:48:32 +0200 Subject: [PATCH 458/598] gh-108765: Python.h no longer includes (#108781) Remove also the HAVE_IEEEFP_H macro: remove ieeefp.h from the AC_CHECK_HEADERS() check of configure.ac. --- Doc/whatsnew/3.13.rst | 6 ++++++ Include/Python.h | 1 + Include/pyport.h | 6 ------ .../C API/2023-09-01-20-41-49.gh-issue-108765.5dXc1r.rst | 4 ++++ configure | 6 ------ configure.ac | 2 +- pyconfig.h.in | 3 --- 7 files changed, 12 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-01-20-41-49.gh-issue-108765.5dXc1r.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d35d20ebb25293..e7b60ddbdbbfda 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -915,6 +915,12 @@ New Features Porting to Python 3.13 ---------------------- +* ``Python.h`` no longer includes the ```` standard header. It was + included for the ``finite()`` function which is now provided by the + ```` header. It should now be included explicitly if needed. Remove + also the ``HAVE_IEEEFP_H`` macro. + (Contributed by Victor Stinner in :gh:`108765`.) + Deprecated ---------- diff --git a/Include/Python.h b/Include/Python.h index 44f0fd3ee4b56b..002a79dbdc9362 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -20,6 +20,7 @@ #include // tolower() #include // uintptr_t #include // INT_MAX +#include // HUGE_VAL #include // va_list #include // wchar_t #ifdef HAVE_STDDEF_H diff --git a/Include/pyport.h b/Include/pyport.h index 67164328d29576..f2046de2bbcc5a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -184,12 +184,6 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_MEMCPY memcpy #endif -#ifdef HAVE_IEEEFP_H -#include /* needed for 'finite' declaration on some platforms */ -#endif - -#include /* Moved here from the math section, before extern "C" */ - /******************************************** * WRAPPER FOR and/or * ********************************************/ diff --git a/Misc/NEWS.d/next/C API/2023-09-01-20-41-49.gh-issue-108765.5dXc1r.rst b/Misc/NEWS.d/next/C API/2023-09-01-20-41-49.gh-issue-108765.5dXc1r.rst new file mode 100644 index 00000000000000..cc512df7e1bc08 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-01-20-41-49.gh-issue-108765.5dXc1r.rst @@ -0,0 +1,4 @@ +``Python.h`` no longer includes the ```` standard header. It was +included for the ``finite()`` function which is now provided by the +```` header. It should now be included explicitly if needed. Remove +also the ``HAVE_IEEEFP_H`` macro. Patch by Victor Stinner. diff --git a/configure b/configure index 983b1edf4a86e1..d73b4b271ac719 100755 --- a/configure +++ b/configure @@ -10325,12 +10325,6 @@ if test "x$ac_cv_header_grp_h" = xyes then : printf "%s\n" "#define HAVE_GRP_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "ieeefp.h" "ac_cv_header_ieeefp_h" "$ac_includes_default" -if test "x$ac_cv_header_ieeefp_h" = xyes -then : - printf "%s\n" "#define HAVE_IEEEFP_H 1" >>confdefs.h - fi ac_fn_c_check_header_compile "$LINENO" "io.h" "ac_cv_header_io_h" "$ac_includes_default" if test "x$ac_cv_header_io_h" = xyes diff --git a/configure.ac b/configure.ac index 10f132faa9d80d..612c072af329d4 100644 --- a/configure.ac +++ b/configure.ac @@ -2667,7 +2667,7 @@ AC_DEFINE([STDC_HEADERS], [1], # checks for header files AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ - ieeefp.h io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ + io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ linux/random.h linux/soundcard.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 418ccade8e8bb1..86c72cc6b4e62a 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -607,9 +607,6 @@ /* Define this if you have le64toh() */ #undef HAVE_HTOLE64 -/* Define to 1 if you have the header file. */ -#undef HAVE_IEEEFP_H - /* Define to 1 if you have the `if_nameindex' function. */ #undef HAVE_IF_NAMEINDEX From 4f9b706c6f5d4422a398146bfd011daedaef1851 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 16:42:07 +0200 Subject: [PATCH 459/598] gh-108794: doctest counts skipped tests (#108795) * Add 'skipped' attribute to TestResults. * Add 'skips' attribute to DocTestRunner. * Rename private DocTestRunner._name2ft attribute to DocTestRunner._stats. * Use f-string for string formatting. * Add some tests. * Document DocTestRunner attributes and its API for statistics. * Document TestResults class. Co-authored-by: Alex Waygood --- Doc/library/doctest.rst | 49 +++++- Doc/whatsnew/3.13.rst | 8 + Lib/doctest.py | 150 ++++++++++-------- Lib/test/test_doctest.py | 32 ++++ ...-09-02-05-13-38.gh-issue-108794.tGHXBt.rst | 3 + 5 files changed, 175 insertions(+), 67 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 92da6133f9bf09..54a8e79a0ef941 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1409,6 +1409,27 @@ DocTestParser objects identifying this string, and is only used for error messages. +TestResults objects +^^^^^^^^^^^^^^^^^^^ + + +.. class:: TestResults(failed, attempted) + + .. attribute:: failed + + Number of failed tests. + + .. attribute:: attempted + + Number of attempted tests. + + .. attribute:: skipped + + Number of skipped tests. + + .. versionadded:: 3.13 + + .. _doctest-doctestrunner: DocTestRunner objects @@ -1427,7 +1448,7 @@ DocTestRunner objects passing a subclass of :class:`OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. First, an output - function can be passed to :meth:`TestRunner.run`; this function will be called + function can be passed to :meth:`run`; this function will be called with strings that should be displayed. It defaults to ``sys.stdout.write``. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods @@ -1448,6 +1469,10 @@ DocTestRunner objects runner compares expected output to actual output, and how it displays failures. For more information, see section :ref:`doctest-options`. + The test runner accumulates statistics. The aggregated number of attempted, + failed and skipped examples is also available via the :attr:`tries`, + :attr:`failures` and :attr:`skips` attributes. The :meth:`run` and + :meth:`summarize` methods return a :class:`TestResults` instance. :class:`DocTestParser` defines the following methods: @@ -1500,7 +1525,8 @@ DocTestRunner objects .. method:: run(test, compileflags=None, out=None, clear_globs=True) Run the examples in *test* (a :class:`DocTest` object), and display the - results using the writer function *out*. + results using the writer function *out*. Return a :class:`TestResults` + instance. The examples are run in the namespace ``test.globs``. If *clear_globs* is true (the default), then this namespace will be cleared after the test runs, @@ -1519,12 +1545,29 @@ DocTestRunner objects .. method:: summarize(verbose=None) Print a summary of all the test cases that have been run by this DocTestRunner, - and return a :term:`named tuple` ``TestResults(failed, attempted)``. + and return a :class:`TestResults` instance. The optional *verbose* argument controls how detailed the summary is. If the verbosity is not specified, then the :class:`DocTestRunner`'s verbosity is used. + :class:`DocTestParser` has the following attributes: + + .. attribute:: tries + + Number of attempted examples. + + .. attribute:: failures + + Number of failed examples. + + .. attribute:: skips + + Number of skipped examples. + + .. versionadded:: 3.13 + + .. _doctest-outputchecker: OutputChecker objects diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e7b60ddbdbbfda..eb3d8323daaf0e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -122,6 +122,14 @@ dbm from the database. (Contributed by Dong-hee Na in :gh:`107122`.) +doctest +------- + +* The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped + tests. Add :attr:`doctest.DocTestRunner.skips` and + :attr:`doctest.TestResults.skipped` attributes. + (Contributed by Victor Stinner in :gh:`108794`.) + io -- diff --git a/Lib/doctest.py b/Lib/doctest.py index a63df46a112e64..46b4dd6769b063 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -105,7 +105,23 @@ def _test(): from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple -TestResults = namedtuple('TestResults', 'failed attempted') + +class TestResults(namedtuple('TestResults', 'failed attempted')): + def __new__(cls, failed, attempted, *, skipped=0): + results = super().__new__(cls, failed, attempted) + results.skipped = skipped + return results + + def __repr__(self): + if self.skipped: + return (f'TestResults(failed={self.failed}, ' + f'attempted={self.attempted}, ' + f'skipped={self.skipped})') + else: + # Leave the repr() unchanged for backward compatibility + # if skipped is zero + return super().__repr__() + # There are 4 basic classes: # - Example: a pair, plus an intra-docstring line number. @@ -1150,8 +1166,7 @@ class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. + returns a TestResults instance. >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1164,8 +1179,8 @@ class DocTestRunner: _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: + have been run by the runner, and returns an aggregated TestResults + instance: >>> runner.summarize(verbose=1) 4 items passed all tests: @@ -1178,13 +1193,15 @@ class DocTestRunner: Test passed. TestResults(failed=0, attempted=7) - The aggregated number of tried examples and failed examples is - also available via the `tries` and `failures` attributes: + The aggregated number of tried examples and failed examples is also + available via the `tries`, `failures` and `skips` attributes: >>> runner.tries 7 >>> runner.failures 0 + >>> runner.skips + 0 The comparison between expected outputs and actual outputs is done by an `OutputChecker`. This comparison may be customized with a @@ -1233,7 +1250,8 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Keep track of the examples we've run. self.tries = 0 self.failures = 0 - self._name2ft = {} + self.skips = 0 + self._stats = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() @@ -1302,13 +1320,11 @@ def __run(self, test, compileflags, out): Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_*` methods, using the writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. + flags that should be used to execute examples. Return a TestResults + instance. The examples are run in the namespace `test.globs`. """ - # Keep track of the number of failures and tries. - failures = tries = 0 + # Keep track of the number of failed, attempted, skipped examples. + failures = attempted = skips = 0 # Save the option flags (since option directives can be used # to modify them). @@ -1320,6 +1336,7 @@ def __run(self, test, compileflags, out): # Process each example. for examplenum, example in enumerate(test.examples): + attempted += 1 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. @@ -1337,10 +1354,10 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + skips += 1 continue # Record that we started this example. - tries += 1 if not quiet: self.report_start(out, test, example) @@ -1418,19 +1435,22 @@ def __run(self, test, compileflags, out): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return TestResults(failures, tries) + # Record and return the number of failures and attempted. + self.__record_outcome(test, failures, attempted, skips) + return TestResults(failures, attempted, skipped=skips) - def __record_outcome(self, test, f, t): + def __record_outcome(self, test, failures, tries, skips): """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. + Record the fact that the given DocTest (`test`) generated `failures` + failures out of `tries` tried examples. """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t + failures2, tries2, skips2 = self._stats.get(test.name, (0, 0, 0)) + self._stats[test.name] = (failures + failures2, + tries + tries2, + skips + skips2) + self.failures += failures + self.tries += tries + self.skips += skips __LINECACHE_FILENAME_RE = re.compile(r'.+)' @@ -1519,9 +1539,7 @@ def out(s): def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. + this DocTestRunner, and return a TestResults instance. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the @@ -1532,59 +1550,61 @@ def summarize(self, verbose=None): notests = [] passed = [] failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: + total_tries = total_failures = total_skips = 0 + for item in self._stats.items(): + name, (failures, tries, skips) = item + assert failures <= tries + total_tries += tries + total_failures += failures + total_skips += skips + if tries == 0: notests.append(name) - elif f == 0: - passed.append( (name, t) ) + elif failures == 0: + passed.append((name, tries)) else: - failed.append(x) + failed.append(item) if verbose: if notests: - print(len(notests), "items had no tests:") + print(f"{len(notests)} items had no tests:") notests.sort() - for thing in notests: - print(" ", thing) + for name in notests: + print(f" {name}") if passed: - print(len(passed), "items passed all tests:") + print(f"{len(passed)} items passed all tests:") passed.sort() - for thing, count in passed: - print(" %3d tests in %s" % (count, thing)) + for name, count in passed: + print(f" {count:3d} tests in {name}") if failed: print(self.DIVIDER) - print(len(failed), "items had failures:") + print(f"{len(failed)} items had failures:") failed.sort() - for thing, (f, t) in failed: - print(" %3d of %3d in %s" % (f, t, thing)) + for name, (failures, tries, skips) in failed: + print(f" {failures:3d} of {tries:3d} in {name}") if verbose: - print(totalt, "tests in", len(self._name2ft), "items.") - print(totalt - totalf, "passed and", totalf, "failed.") - if totalf: - print("***Test Failed***", totalf, "failures.") + print(f"{total_tries} tests in {len(self._stats)} items.") + print(f"{total_tries - total_failures} passed and {total_failures} failed.") + if total_failures: + msg = f"***Test Failed*** {total_failures} failures" + if total_skips: + msg = f"{msg} and {total_skips} skipped tests" + print(f"{msg}.") elif verbose: print("Test passed.") - return TestResults(totalf, totalt) + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): + d = self._stats + for name, (failures, tries, skips) in other._stats.items(): if name in d: - # Don't print here by default, since doing - # so breaks some of the buildbots - #print("*** DocTestRunner.merge: '" + name + "' in both" \ - # " testers; summing outcomes.") - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t + failures2, tries2, skips2 = d[name] + failures = failures + failures2 + tries = tries + tries2 + skips = skips + skips2 + d[name] = (failures, tries, skips) + class OutputChecker: """ @@ -1984,7 +2004,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, @@ -2107,7 +2128,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index c364e81a6b1495..9cc460c8b913f6 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -748,6 +748,38 @@ def non_Python_modules(): r""" """ +class TestDocTest(unittest.TestCase): + + def test_run(self): + test = ''' + >>> 1 + 1 + 11 + >>> 2 + 3 # doctest: +SKIP + "23" + >>> 5 + 7 + 57 + ''' + + def myfunc(): + pass + myfunc.__doc__ = test + + # test DocTestFinder.run() + test = doctest.DocTestFinder().find(myfunc)[0] + with support.captured_stdout(): + with support.captured_stderr(): + results = doctest.DocTestRunner(verbose=False).run(test) + + # test TestResults + self.assertIsInstance(results, doctest.TestResults) + self.assertEqual(results.failed, 2) + self.assertEqual(results.attempted, 3) + self.assertEqual(results.skipped, 1) + self.assertEqual(tuple(results), (2, 3)) + x, y = results + self.assertEqual((x, y), (2, 3)) + + class TestDocTestFinder(unittest.TestCase): def test_issue35753(self): diff --git a/Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst b/Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst new file mode 100644 index 00000000000000..00027c7c07da2e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst @@ -0,0 +1,3 @@ +The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped +tests. Add :attr:`doctest.DocTestRunner.skips` and +:attr:`doctest.TestResults.skipped` attributes. Patch by Victor Stinner. From 594b00057e667e0d8d4e41748be056cdd829e919 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 16:50:18 +0200 Subject: [PATCH 460/598] gh-108765: Python.h no longer includes (#108783) --- Doc/whatsnew/3.13.rst | 5 +++++ Include/Python.h | 11 +++++----- ...-09-01-21-10-29.gh-issue-108765.eeXtYF.rst | 4 ++++ Modules/_ctypes/malloc_closure.c | 13 ++++++----- Modules/_posixsubprocess.c | 22 +++++++++---------- Modules/_testcapimodule.c | 3 --- Modules/grpmodule.c | 3 ++- Modules/mmapmodule.c | 3 +++ Modules/posixmodule.c | 2 +- Modules/pwdmodule.c | 3 ++- Modules/resource.c | 11 +++++----- Modules/selectmodule.c | 3 +++ Modules/socketmodule.c | 2 +- Programs/_freeze_module.c | 2 +- Python/dup2.c | 6 ++--- Python/perf_trampoline.c | 4 ++-- 16 files changed, 55 insertions(+), 42 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-01-21-10-29.gh-issue-108765.eeXtYF.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index eb3d8323daaf0e..401de11b34ec5f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -929,6 +929,11 @@ Porting to Python 3.13 also the ``HAVE_IEEEFP_H`` macro. (Contributed by Victor Stinner in :gh:`108765`.) +* ``Python.h`` no longer includes the ```` standard header file. If + needed, it should now be included explicitly. For example, it provides the + functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. + (Contributed by Victor Stinner in :gh:`108765`.) + Deprecated ---------- diff --git a/Include/Python.h b/Include/Python.h index 002a79dbdc9362..4cc72bb23ce7a3 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -26,14 +26,13 @@ #ifdef HAVE_STDDEF_H # include // size_t #endif -#ifndef MS_WINDOWS -# include // sysconf() +#ifdef HAVE_SYS_TYPES_H +# include // ssize_t #endif -// errno.h, stdio.h, stdlib.h and string.h headers are no longer used by Python -// headers, but kept for backward compatibility (no introduce new compiler -// warnings). They are not included by the limited C API version 3.11 and -// above. +// errno.h, stdio.h, stdlib.h and string.h headers are no longer used by +// Python, but kept for backward compatibility (avoid compiler warnings). +// They are no longer included by limited C API version 3.11 and newer. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # include // errno # include // FILE* diff --git a/Misc/NEWS.d/next/C API/2023-09-01-21-10-29.gh-issue-108765.eeXtYF.rst b/Misc/NEWS.d/next/C API/2023-09-01-21-10-29.gh-issue-108765.eeXtYF.rst new file mode 100644 index 00000000000000..ff8f79998fa968 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-01-21-10-29.gh-issue-108765.eeXtYF.rst @@ -0,0 +1,4 @@ +``Python.h`` no longer includes the ```` standard header file. If +needed, it should now be included explicitly. For example, it provides the +functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. +Patch by Victor Stinner. diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index 3a859322772ba7..bb4f8f21bd3f77 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -1,16 +1,17 @@ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif + #include #include #ifdef MS_WIN32 -#include +# include #else -#include -#include -# if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -# define MAP_ANONYMOUS MAP_ANON -# endif +# include +# include // sysconf() +# if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +# define MAP_ANONYMOUS MAP_ANON +# endif #endif #include "ctypes.h" diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index ac2b0d4f55468c..ef76d26282e1b3 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -8,28 +8,28 @@ #include "pycore_pystate.h" #include "pycore_signal.h" // _Py_RestoreSignals() #if defined(HAVE_PIPE2) && !defined(_GNU_SOURCE) -# define _GNU_SOURCE +# define _GNU_SOURCE #endif -#include -#include +#include // close() +#include // fcntl() #ifdef HAVE_SYS_TYPES_H -#include +# include #endif #if defined(HAVE_SYS_STAT_H) -#include +# include // stat() #endif #ifdef HAVE_SYS_SYSCALL_H -#include +# include #endif #if defined(HAVE_SYS_RESOURCE_H) -#include +# include #endif #ifdef HAVE_DIRENT_H -#include +# include // opendir() +#endif +#if defined(HAVE_SETGROUPS) +# include // setgroups() #endif -#ifdef HAVE_GRP_H -#include -#endif /* HAVE_GRP_H */ #include "posixmodule.h" diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 4fc354ae79bfed..ab33702cdfd872 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -24,9 +24,6 @@ #include // FLT_MAX #include #include // offsetof() -#ifndef MS_WINDOWS -# include -#endif #ifdef HAVE_SYS_WAIT_H # include // W_STOPCODE diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index f5709296334a8f..20e83de84e8340 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -4,7 +4,8 @@ #include "Python.h" #include "posixmodule.h" -#include +#include // getgrgid_r() +#include // sysconf() #include "clinic/grpmodule.c.h" /*[clinic input] diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index c8cd7e59dbab50..d11200a4042551 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -28,6 +28,9 @@ #include "pycore_fileutils.h" // _Py_stat_struct #include // offsetof() +#ifndef MS_WINDOWS +# include // close() +#endif // to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA // need to be replaced with OpenFileMappingW / CreateFileMappingW diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 761542866d8f96..6e829b200fa46d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -286,7 +286,7 @@ corresponding Unix manual entries for more information on calls."); #endif #ifdef HAVE_COPY_FILE_RANGE -# include +# include // copy_file_range() #endif #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index cc2e2a43893971..b7034369c4731e 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -4,7 +4,8 @@ #include "Python.h" #include "posixmodule.h" -#include +#include // getpwuid() +#include // sysconf() #include "clinic/pwdmodule.c.h" /*[clinic input] diff --git a/Modules/resource.c b/Modules/resource.c index 4614f5e98cc888..f5d9972d9a8ff7 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,13 +1,12 @@ - #include "Python.h" -#include +#include // errno +#include +#include // getrusage() #ifdef HAVE_SYS_TIME_H -#include +# include #endif #include -#include -#include -#include +#include // getpagesize() /* On some systems, these aren't in any header file. On others they are, with inconsistent prototypes. diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 4987cf0f2065c2..c56e682b21e2a1 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -17,6 +17,9 @@ #include "pycore_time.h" // _PyTime_t #include // offsetof() +#ifndef MS_WINDOWS +# include // close() +#endif #ifdef HAVE_SYS_DEVPOLL_H #include diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 2f12c9cedbd8a6..74b1c1c661604f 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -269,7 +269,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #ifdef HAVE_NETDB_H # include #endif -# include +#include // close() /* Headers needed for inet_ntoa() and inet_addr() */ # include diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index e55f1d56745c4d..f6c46fa629efba 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -19,7 +19,7 @@ #include #include #ifndef MS_WINDOWS -#include +# include #endif uint32_t _Py_next_func_version = 1; diff --git a/Python/dup2.c b/Python/dup2.c index a1df0492099163..936211f27ec737 100644 --- a/Python/dup2.c +++ b/Python/dup2.c @@ -11,9 +11,9 @@ * Return fd2 if all went well; return BADEXIT otherwise. */ -#include -#include -#include +#include // errno +#include // fcntl() +#include // close() #define BADEXIT -1 diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index b8885a303977d0..10675bf9f8292a 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -140,9 +140,9 @@ any DWARF information available for them). #include #include #include -#include +#include // mmap() #include -#include +#include // sysconf() #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__) #define PY_HAVE_INVALIDATE_ICACHE From bdc3c884cdc90102ad68b6b55dc9b988e729ae35 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 2 Sep 2023 16:08:03 +0100 Subject: [PATCH 461/598] GH-78722: Raise exceptions from `pathlib.Path.iterdir()` without delay. (#107320) `pathlib.Path.iterdir()` now immediately raises any `OSError` exception from `os.listdir()`, rather than waiting until its result is iterated over. --- Lib/pathlib.py | 3 +-- Lib/test/test_pathlib.py | 2 +- .../next/Library/2023-07-26-22-52-48.gh-issue-78722.6SKBLt.rst | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-26-22-52-48.gh-issue-78722.6SKBLt.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 758f70f5ba7077..f4ec315da6b4fa 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1009,8 +1009,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - for name in os.listdir(self): - yield self._make_child_relpath(name) + return (self._make_child_relpath(name) for name in os.listdir(self)) def _scandir(self): # bpo-24132: a future version of pathlib will support subclassing of diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 74deec84336f72..09df3fe471fc3e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1766,7 +1766,7 @@ def test_iterdir_nodir(self): # __iter__ on something that is not a directory. p = self.cls(BASE, 'fileA') with self.assertRaises(OSError) as cm: - next(p.iterdir()) + p.iterdir() # ENOENT or EINVAL under Windows, ENOTDIR otherwise # (see issue #12802). self.assertIn(cm.exception.errno, (errno.ENOTDIR, diff --git a/Misc/NEWS.d/next/Library/2023-07-26-22-52-48.gh-issue-78722.6SKBLt.rst b/Misc/NEWS.d/next/Library/2023-07-26-22-52-48.gh-issue-78722.6SKBLt.rst new file mode 100644 index 00000000000000..aea26ee2f99467 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-26-22-52-48.gh-issue-78722.6SKBLt.rst @@ -0,0 +1,2 @@ +Fix issue where :meth:`pathlib.Path.iterdir` did not raise :exc:`OSError` +until iterated. From e7de0c5901b85a5241386a33f98c27a4e08d5384 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 17:51:19 +0200 Subject: [PATCH 462/598] gh-108765: Python.h no longer includes (#108775) Python.h no longer includes , and standard header files. * Add include to xxsubtype.c. * Add include to posixmodule.c and semaphore.c. * readline.c includes instead of . * resource.c no longer includes and . --- Doc/whatsnew/3.13.rst | 8 ++++++++ Include/pyport.h | 19 ------------------- ...-09-01-18-42-31.gh-issue-108765.IyYNDu.rst | 6 ++++++ Modules/_multiprocessing/semaphore.c | 4 ++++ Modules/posixmodule.c | 4 ++++ Modules/readline.c | 16 ++++++++-------- Modules/resource.c | 4 ---- Modules/signalmodule.c | 10 +++++----- Modules/timemodule.c | 2 +- Modules/xxsubtype.c | 2 ++ Python/pytime.c | 5 +++++ 11 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-01-18-42-31.gh-issue-108765.IyYNDu.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 401de11b34ec5f..1c91a1dadb899f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -934,6 +934,14 @@ Porting to Python 3.13 functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. (Contributed by Victor Stinner in :gh:`108765`.) +* ``Python.h`` no longer includes these standard header files: ````, + ```` and ````. If needed, they should now be + included explicitly. For example, ```` provides the ``clock()`` and + ``gmtime()`` functions, ```` provides the ``select()`` + function, and ```` provides the ``futimes()``, ``gettimeofday()`` + and ``setitimer()`` functions. + (Contributed by Victor Stinner in :gh:`108765`.) + Deprecated ---------- diff --git a/Include/pyport.h b/Include/pyport.h index f2046de2bbcc5a..c4168d10f58151 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -184,25 +184,6 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_MEMCPY memcpy #endif -/******************************************** - * WRAPPER FOR and/or * - ********************************************/ - -#ifdef HAVE_SYS_TIME_H -#include -#endif -#include - -/****************************** - * WRAPPER FOR * - ******************************/ - -/* NB caller must include */ - -#ifdef HAVE_SYS_SELECT_H -#include -#endif /* !HAVE_SYS_SELECT_H */ - /******************************* * stat() and fstat() fiddling * *******************************/ diff --git a/Misc/NEWS.d/next/C API/2023-09-01-18-42-31.gh-issue-108765.IyYNDu.rst b/Misc/NEWS.d/next/C API/2023-09-01-18-42-31.gh-issue-108765.IyYNDu.rst new file mode 100644 index 00000000000000..7b33481f225b5a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-01-18-42-31.gh-issue-108765.IyYNDu.rst @@ -0,0 +1,6 @@ +``Python.h`` no longer includes these standard header files: ````, +```` and ````. If needed, they should now be included +explicitly. For example, ```` provides the ``clock()`` and ``gmtime()`` +functions, ```` provides the ``select()`` function, and +```` provides the ``futimes()``, ``gettimeofday()`` and +``setitimer()`` functions. Patch by Victor Stinner. diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index d22b8d18e33e42..f8f2afda28d06d 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -9,6 +9,10 @@ #include "multiprocessing.h" +#ifdef HAVE_SYS_TIME_H +# include // gettimeofday() +#endif + #ifdef HAVE_MP_SEMAPHORE enum { RECURSIVE_MUTEX, SEMAPHORE }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6e829b200fa46d..b4c502bef50ba9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -57,6 +57,10 @@ #include // ctermid() #include // system() +#ifdef HAVE_SYS_TIME_H +# include // futimes() +#endif + // SGI apparently needs this forward declaration #ifdef HAVE__GETPTY diff --git a/Modules/readline.c b/Modules/readline.c index 2531b236b6ef17..aeae654162f13f 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -12,14 +12,13 @@ #include "Python.h" #include "pycore_pylifecycle.h" // _Py_SetLocaleFromEnv() -#include -#include -#include +#include // errno +#include // SIGWINCH #include // free() -#ifdef HAVE_SYS_TIME_H -#include +#include // strdup() +#ifdef HAVE_SYS_SELECT_H +# include // select() #endif -#include #if defined(HAVE_SETLOCALE) /* GNU readline() mistakenly sets the LC_CTYPE locale. @@ -27,7 +26,7 @@ * We must save and restore the locale around the rl_initialize() call. */ #define SAVE_LOCALE -#include +# include // setlocale() #endif #ifdef SAVE_LOCALE @@ -1333,7 +1332,8 @@ readline_until_enter_or_signal(const char *prompt, int *signal) int has_input = 0, err = 0; while (!has_input) - { struct timeval timeout = {0, 100000}; /* 0.1 seconds */ + { + struct timeval timeout = {0, 100000}; // 100 ms (0.1 seconds) /* [Bug #1552726] Only limit the pause if an input hook has been defined. */ diff --git a/Modules/resource.c b/Modules/resource.c index f5d9972d9a8ff7..9e302a3a1ed962 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -2,10 +2,6 @@ #include // errno #include #include // getrusage() -#ifdef HAVE_SYS_TIME_H -# include -#endif -#include #include // getpagesize() /* On some systems, these aren't in any header file. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 3adb2e8dfe58d8..8d6556727b3a5a 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -16,10 +16,10 @@ #include "pycore_signal.h" // _Py_RestoreSignals() #ifndef MS_WINDOWS -# include "posixmodule.h" +# include "posixmodule.h" // _PyLong_FromUid() #endif #ifdef MS_WINDOWS -# include "socketmodule.h" /* needed for SOCKET_T */ +# include "socketmodule.h" // SOCKET_T #endif #ifdef MS_WINDOWS @@ -29,16 +29,16 @@ #endif #ifdef HAVE_SIGNAL_H -# include +# include // sigaction() #endif #ifdef HAVE_SYS_SYSCALL_H -# include +# include // __NR_pidfd_send_signal #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef HAVE_SYS_TIME_H -# include +# include // setitimer() #endif #if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 68948b6be1a61a..4e55da71b117ca 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -7,7 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include - +#include // clock() #ifdef HAVE_SYS_TIMES_H # include #endif diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 63b22268c597b6..560f43e5b3a643 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -1,5 +1,7 @@ #include "Python.h" + #include // offsetof() +#include // clock() PyDoc_STRVAR(xxsubtype__doc__, diff --git a/Python/pytime.c b/Python/pytime.c index 49cd5f4e8ea617..d1e29e57d362f6 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,10 @@ #include "Python.h" #include "pycore_time.h" // _PyTime_t + +#include // gmtime_r() +#ifdef HAVE_SYS_TIME_H +# include // gettimeofday() +#endif #ifdef MS_WINDOWS # include // struct timeval #endif From d4e534cbb35678c82b3a1276826af55d7bfc23b6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 18:09:36 +0200 Subject: [PATCH 463/598] regrtest computes statistics (#108793) test_netrc, test_pep646_syntax and test_xml_etree now return results in the test_main() function. Changes: * Rewrite TestResult as a dataclass with a new State class. * Add test.support.TestStats class and Regrtest.stats_dict attribute. * libregrtest.runtest functions now modify a TestResult instance in-place. * libregrtest summary lists the number of run tests and skipped tests, and denied resources. * Add TestResult.has_meaningful_duration() method. * Compute TestResult duration in the upper function. * Use time.perf_counter() instead of time.monotonic(). * Regrtest: rename 'resource_denieds' attribute to 'resource_denied'. * Rename CHILD_ERROR to MULTIPROCESSING_ERROR. * Use match/case syntadx to have different code depending on the test state. Co-authored-by: Alex Waygood --- Lib/test/libregrtest/main.py | 126 +++++++---- Lib/test/libregrtest/refleak.py | 5 +- Lib/test/libregrtest/runtest.py | 324 ++++++++++++++++------------- Lib/test/libregrtest/runtest_mp.py | 84 ++++---- Lib/test/libregrtest/save_env.py | 8 +- Lib/test/support/__init__.py | 61 ++++-- Lib/test/test_netrc.py | 2 +- Lib/test/test_pep646_syntax.py | 2 +- Lib/test/test_regrtest.py | 206 +++++++++++++----- Lib/test/test_xml_etree.py | 2 +- 10 files changed, 512 insertions(+), 308 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3d290c849b43ed..6e6423e156781b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,15 +11,14 @@ import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, split_test_packages, runtest, get_abs_module, is_failed, - PROGRESS_MIN_TIME, - Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted, - ChildError, DidNotRun) + findtests, split_test_packages, runtest, get_abs_module, + PROGRESS_MIN_TIME, State) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (removepy, count, format_duration, printlist, get_build_info) from test import support +from test.support import TestStats from test.support import os_helper from test.support import threading_helper @@ -78,13 +77,14 @@ def __init__(self): self.good = [] self.bad = [] self.skipped = [] - self.resource_denieds = [] + self.resource_denied = [] self.environment_changed = [] self.run_no_tests = [] self.need_rerun = [] self.rerun = [] self.first_result = None self.interrupted = False + self.stats_dict: dict[str, TestStats] = {} # used by --slow self.test_times = [] @@ -93,7 +93,7 @@ def __init__(self): self.tracer = None # used to display the progress bar "[ 3/100]" - self.start_time = time.monotonic() + self.start_time = time.perf_counter() self.test_count = '' self.test_count_width = 1 @@ -111,36 +111,41 @@ def __init__(self): def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) - | set(self.resource_denieds) | set(self.environment_changed) + | set(self.resource_denied) | set(self.environment_changed) | set(self.run_no_tests)) def accumulate_result(self, result, rerun=False): - test_name = result.name - - if not isinstance(result, (ChildError, Interrupted)) and not rerun: - self.test_times.append((result.duration_sec, test_name)) - - if isinstance(result, Passed): - self.good.append(test_name) - elif isinstance(result, ResourceDenied): - self.skipped.append(test_name) - self.resource_denieds.append(test_name) - elif isinstance(result, Skipped): - self.skipped.append(test_name) - elif isinstance(result, EnvChanged): - self.environment_changed.append(test_name) - elif isinstance(result, Failed): - if not rerun: - self.bad.append(test_name) - self.need_rerun.append(result) - elif isinstance(result, DidNotRun): - self.run_no_tests.append(test_name) - elif isinstance(result, Interrupted): - self.interrupted = True - else: - raise ValueError("invalid test result: %r" % result) + test_name = result.test_name + + if result.has_meaningful_duration() and not rerun: + self.test_times.append((result.duration, test_name)) - if rerun and not isinstance(result, (Failed, Interrupted)): + match result.state: + case State.PASSED: + self.good.append(test_name) + case State.ENV_CHANGED: + self.environment_changed.append(test_name) + case State.SKIPPED: + self.skipped.append(test_name) + case State.RESOURCE_DENIED: + self.skipped.append(test_name) + self.resource_denied.append(test_name) + case State.INTERRUPTED: + self.interrupted = True + case State.DID_NOT_RUN: + self.run_no_tests.append(test_name) + case _: + if result.is_failed(self.ns.fail_env_changed): + if not rerun: + self.bad.append(test_name) + self.need_rerun.append(result) + else: + raise ValueError(f"invalid test state: {state!r}") + + if result.stats is not None: + self.stats_dict[result.test_name] = result.stats + + if rerun and not(result.is_failed(False) or result.state == State.INTERRUPTED): self.bad.remove(test_name) xml_data = result.xml_data @@ -162,7 +167,7 @@ def log(self, line=''): line = f"load avg: {load_avg:.2f} {line}" # add the timestamp prefix: "0:01:05 " - test_time = time.monotonic() - self.start_time + test_time = time.perf_counter() - self.start_time mins, secs = divmod(int(test_time), 60) hours, mins = divmod(mins, 60) @@ -337,7 +342,7 @@ def rerun_failed_tests(self): rerun_list = list(self.need_rerun) self.need_rerun.clear() for result in rerun_list: - test_name = result.name + test_name = result.test_name self.rerun.append(test_name) errors = result.errors or [] @@ -364,7 +369,7 @@ def rerun_failed_tests(self): self.accumulate_result(result, rerun=True) - if isinstance(result, Interrupted): + if result.state == State.INTERRUPTED: break if self.bad: @@ -461,7 +466,7 @@ def run_tests_sequential(self): previous_test = None for test_index, test_name in enumerate(self.tests, 1): - start_time = time.monotonic() + start_time = time.perf_counter() text = test_name if previous_test: @@ -480,14 +485,14 @@ def run_tests_sequential(self): result = runtest(self.ns, test_name) self.accumulate_result(result) - if isinstance(result, Interrupted): + if result.state == State.INTERRUPTED: break previous_test = str(result) - test_time = time.monotonic() - start_time + test_time = time.perf_counter() - start_time if test_time >= PROGRESS_MIN_TIME: previous_test = "%s in %s" % (previous_test, format_duration(test_time)) - elif isinstance(result, Passed): + elif result.state == State.PASSED: # be quiet: say nothing if the test passed shortly previous_test = None @@ -496,7 +501,7 @@ def run_tests_sequential(self): if module not in save_modules and module.startswith("test."): support.unload(module) - if self.ns.failfast and is_failed(result, self.ns): + if self.ns.failfast and result.is_failed(self.ns.fail_env_changed): break if previous_test: @@ -638,13 +643,48 @@ def finalize(self): coverdir=self.ns.coverdir) print() - duration = time.monotonic() - self.start_time - print("Total duration: %s" % format_duration(duration)) - print("Tests result: %s" % self.get_tests_result()) + self.display_summary() if self.ns.runleaks: os.system("leaks %d" % os.getpid()) + def display_summary(self): + duration = time.perf_counter() - self.start_time + + # Total duration + print("Total duration: %s" % format_duration(duration)) + + # Total tests + total = TestStats() + for stats in self.stats_dict.values(): + total.accumulate(stats) + stats = [f'run={total.tests_run:,}'] + if total.failures: + stats.append(f'failures={total.failures:,}') + if total.skipped: + stats.append(f'skipped={total.skipped:,}') + print(f"Total tests: {' '.join(stats)}") + + # Total test files + report = [f'success={len(self.good)}'] + if self.bad: + report.append(f'failed={len(self.bad)}') + if self.environment_changed: + report.append(f'env_changed={len(self.environment_changed)}') + if self.skipped: + report.append(f'skipped={len(self.skipped)}') + if self.resource_denied: + report.append(f'resource_denied={len(self.resource_denied)}') + if self.rerun: + report.append(f'rerun={len(self.rerun)}') + if self.run_no_tests: + report.append(f'run_no_tests={len(self.run_no_tests)}') + print(f"Total test files: {' '.join(report)}") + + # Result + result = self.get_tests_result() + print(f"Result: {result}") + def save_xml_result(self): if not self.ns.xmlpath and not self.testsuite_xml: return diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index cd11d385591f80..206802b60ddcd0 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -83,11 +83,12 @@ def get_pooled_int(value): print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) + results = None dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() for i in rep_range: - test_func() + results = test_func() dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -151,7 +152,7 @@ def check_fd_deltas(deltas): print(msg, file=refrep) refrep.flush() failed = True - return failed + return (failed, results) def dash_R_cleanup(fs, ps, pic, zdc, abcs): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index fd49927679bdea..6fa60697371b72 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -1,3 +1,5 @@ +import dataclasses +import doctest import faulthandler import functools import gc @@ -10,6 +12,7 @@ import unittest from test import support +from test.support import TestStats from test.support import os_helper from test.support import threading_helper from test.libregrtest.cmdline import Namespace @@ -17,108 +20,114 @@ from test.libregrtest.utils import clear_caches, format_duration, print_warning -class TestResult: - def __init__( - self, - name: str, - duration_sec: float = 0.0, - xml_data: list[str] | None = None, - ) -> None: - self.name = name - self.duration_sec = duration_sec - self.xml_data = xml_data - - def __str__(self) -> str: - return f"{self.name} finished" - +# Avoid enum.Enum to reduce the number of imports when tests are run +class State: + PASSED = "PASSED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + UNCAUGHT_EXC = "UNCAUGHT_EXC" + REFLEAK = "REFLEAK" + ENV_CHANGED = "ENV_CHANGED" + RESOURCE_DENIED = "RESOURCE_DENIED" + INTERRUPTED = "INTERRUPTED" + MULTIPROCESSING_ERROR = "MULTIPROCESSING_ERROR" + DID_NOT_RUN = "DID_NOT_RUN" + TIMEOUT = "TIMEOUT" -class Passed(TestResult): - def __str__(self) -> str: - return f"{self.name} passed" - - -class Failed(TestResult): - def __init__( - self, - name: str, - duration_sec: float = 0.0, - xml_data: list[str] | None = None, - errors: list[tuple[str, str]] | None = None, - failures: list[tuple[str, str]] | None = None, - ) -> None: - super().__init__(name, duration_sec=duration_sec, xml_data=xml_data) - self.errors = errors - self.failures = failures + @staticmethod + def is_failed(state): + return state in { + State.FAILED, + State.UNCAUGHT_EXC, + State.REFLEAK, + State.MULTIPROCESSING_ERROR, + State.TIMEOUT} - def __str__(self) -> str: + @staticmethod + def has_meaningful_duration(state): + # Consider that the duration is meaningless for these cases. + # For example, if a whole test file is skipped, its duration + # is unlikely to be the duration of executing its tests, + # but just the duration to execute code which skips the test. + return state not in { + State.SKIPPED, + State.RESOURCE_DENIED, + State.INTERRUPTED, + State.MULTIPROCESSING_ERROR, + State.DID_NOT_RUN} + + +@dataclasses.dataclass(slots=True) +class TestResult: + test_name: str + state: str | None = None + # Test duration in seconds + duration: float | None = None + xml_data: list[str] | None = None + stats: TestStats | None = None + + # errors and failures copied from support.TestFailedWithDetails + errors: list[tuple[str, str]] | None = None + failures: list[tuple[str, str]] | None = None + + def is_failed(self, fail_env_changed: bool) -> bool: + if self.state == State.ENV_CHANGED: + return fail_env_changed + return State.is_failed(self.state) + + def _format_failed(self): if self.errors and self.failures: le = len(self.errors) lf = len(self.failures) error_s = "error" + ("s" if le > 1 else "") failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.name} failed ({le} {error_s}, {lf} {failure_s})" + return f"{self.test_name} failed ({le} {error_s}, {lf} {failure_s})" if self.errors: le = len(self.errors) error_s = "error" + ("s" if le > 1 else "") - return f"{self.name} failed ({le} {error_s})" + return f"{self.test_name} failed ({le} {error_s})" if self.failures: lf = len(self.failures) failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.name} failed ({lf} {failure_s})" - - return f"{self.name} failed" + return f"{self.test_name} failed ({lf} {failure_s})" + return f"{self.test_name} failed" -class UncaughtException(Failed): def __str__(self) -> str: - return f"{self.name} failed (uncaught exception)" - - -class EnvChanged(Failed): - def __str__(self) -> str: - return f"{self.name} failed (env changed)" - - # Convert Passed to EnvChanged - @staticmethod - def from_passed(other): - return EnvChanged(other.name, other.duration_sec, other.xml_data) - - -class RefLeak(Failed): - def __str__(self) -> str: - return f"{self.name} failed (reference leak)" - - -class Skipped(TestResult): - def __str__(self) -> str: - return f"{self.name} skipped" - - -class ResourceDenied(Skipped): - def __str__(self) -> str: - return f"{self.name} skipped (resource denied)" - - -class Interrupted(TestResult): - def __str__(self) -> str: - return f"{self.name} interrupted" - - -class ChildError(Failed): - def __str__(self) -> str: - return f"{self.name} crashed" - - -class DidNotRun(TestResult): - def __str__(self) -> str: - return f"{self.name} ran no tests" - - -class Timeout(Failed): - def __str__(self) -> str: - return f"{self.name} timed out ({format_duration(self.duration_sec)})" + match self.state: + case State.PASSED: + return f"{self.test_name} passed" + case State.FAILED: + return self._format_failed() + case State.SKIPPED: + return f"{self.test_name} skipped" + case State.UNCAUGHT_EXC: + return f"{self.test_name} failed (uncaught exception)" + case State.REFLEAK: + return f"{self.test_name} failed (reference leak)" + case State.ENV_CHANGED: + return f"{self.test_name} failed (env changed)" + case State.RESOURCE_DENIED: + return f"{self.test_name} skipped (resource denied)" + case State.INTERRUPTED: + return f"{self.test_name} interrupted" + case State.MULTIPROCESSING_ERROR: + return f"{self.test_name} process crashed" + case State.DID_NOT_RUN: + return f"{self.test_name} ran no tests" + case State.TIMEOUT: + return f"{self.test_name} timed out ({format_duration(self.duration)})" + case _: + raise ValueError("unknown result state: {state!r}") + + def has_meaningful_duration(self): + return State.has_meaningful_duration(self.state) + + def set_env_changed(self): + if self.state is None or self.state == State.PASSED: + self.state = State.ENV_CHANGED # Minimum duration of a test to display its duration or to mention that @@ -142,12 +151,6 @@ def __str__(self) -> str: FOUND_GARBAGE = [] -def is_failed(result: TestResult, ns: Namespace) -> bool: - if isinstance(result, EnvChanged): - return ns.fail_env_changed - return isinstance(result, Failed) - - def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir @@ -194,9 +197,9 @@ def get_abs_module(ns: Namespace, test_name: str) -> str: return 'test.' + test_name -def _runtest(ns: Namespace, test_name: str) -> TestResult: - # Handle faulthandler timeout, capture stdout+stderr, XML serialization - # and measure time. +def _runtest_capture_output_timeout_junit(result: TestResult, ns: Namespace) -> None: + # Capture stdout and stderr, set faulthandler timeout, + # and create JUnit XML report. output_on_failure = ns.verbose3 @@ -206,7 +209,6 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) - start_time = time.perf_counter() try: support.set_match_tests(ns.match_tests, ns.ignore_tests) support.junit_xml_list = xml_list = [] if ns.xmlpath else None @@ -231,9 +233,9 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: # warnings will be written to sys.stderr below. print_warning.orig_stderr = stream - result = _runtest_inner(ns, test_name, - display_failure=False) - if not isinstance(result, Passed): + _runtest_env_changed_exc(result, ns, display_failure=False) + # Ignore output if the test passed successfully + if result.state != State.PASSED: output = stream.getvalue() finally: sys.stdout = orig_stdout @@ -247,18 +249,13 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: # Tell tests to be moderately quiet support.verbose = ns.verbose - result = _runtest_inner(ns, test_name, - display_failure=not ns.verbose) + _runtest_env_changed_exc(result, ns, + display_failure=not ns.verbose) if xml_list: import xml.etree.ElementTree as ET - result.xml_data = [ - ET.tostring(x).decode('us-ascii') - for x in xml_list - ] - - result.duration_sec = time.perf_counter() - start_time - return result + result.xml_data = [ET.tostring(x).decode('us-ascii') + for x in xml_list] finally: if use_timeout: faulthandler.cancel_dump_traceback_later() @@ -271,19 +268,23 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: ns -- regrtest namespace of options test_name -- the name of the test - Returns a TestResult sub-class depending on the kind of result received. + Returns a TestResult. If ns.xmlpath is not None, xml_data is a list containing each generated testsuite element. """ + start_time = time.perf_counter() + result = TestResult(test_name) try: - return _runtest(ns, test_name) + _runtest_capture_output_timeout_junit(result, ns) except: if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) - return Failed(test_name) + result.state = State.UNCAUGHT_EXC + result.duration = time.perf_counter() - start_time + return result def _test_module(the_module): @@ -293,18 +294,48 @@ def _test_module(the_module): print(error, file=sys.stderr) if loader.errors: raise Exception("errors while loading tests") - support.run_unittest(tests) + return support.run_unittest(tests) def save_env(ns: Namespace, test_name: str): return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) -def _runtest_inner2(ns: Namespace, test_name: str) -> bool: - # Load the test function, run the test function, handle huntrleaks - # to detect leaks. +def regrtest_runner(result, test_func, ns) -> None: + # Run test_func(), collect statistics, and detect reference and memory + # leaks. + + if ns.huntrleaks: + from test.libregrtest.refleak import dash_R + refleak, test_result = dash_R(ns, result.test_name, test_func) + else: + test_result = test_func() + refleak = False + + if refleak: + result.state = State.REFLEAK + + match test_result: + case TestStats(): + stats = test_result + case unittest.TestResult(): + stats = TestStats.from_unittest(test_result) + case doctest.TestResults(): + stats = TestStats.from_doctest(test_result) + case None: + print_warning(f"{result.test_name} test runner returned None: {test_func}") + stats = None + case _: + print_warning(f"Unknown test result type: {type(test_result)}") + stats = None + + result.stats = stats + - abstest = get_abs_module(ns, test_name) +def _load_run_test(result: TestResult, ns: Namespace) -> None: + # Load the test function, run the test function. + + abstest = get_abs_module(ns, result.test_name) # remove the module from sys.module to reload it if it was already imported try: @@ -314,23 +345,15 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: the_module = importlib.import_module(abstest) - if ns.huntrleaks: - from test.libregrtest.refleak import dash_R - # If the test has a test_main, that will run the appropriate # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - test_runner = functools.partial(_test_module, the_module) + test_func = getattr(the_module, "test_main", None) + if test_func is None: + test_func = functools.partial(_test_module, the_module) try: - with save_env(ns, test_name): - if ns.huntrleaks: - # Return True if the test leaked references - refleak = dash_R(ns, test_name, test_runner) - else: - test_runner() - refleak = False + with save_env(ns, result.test_name): + regrtest_runner(result, test_func, ns) finally: # First kill any dangling references to open files etc. # This can also issue some ResourceWarnings which would otherwise get @@ -338,11 +361,11 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: # failures. support.gc_collect() - cleanup_test_droppings(test_name, ns.verbose) + cleanup_test_droppings(result.test_name, ns.verbose) if gc.garbage: support.environment_altered = True - print_warning(f"{test_name} created {len(gc.garbage)} " + print_warning(f"{result.test_name} created {len(gc.garbage)} " f"uncollectable object(s).") # move the uncollectable objects somewhere, @@ -352,12 +375,9 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: support.reap_children() - return refleak - -def _runtest_inner( - ns: Namespace, test_name: str, display_failure: bool = True -) -> TestResult: +def _runtest_env_changed_exc(result: TestResult, ns: Namespace, + display_failure: bool = True) -> None: # Detect environment changes, handle exceptions. # Reset the environment_altered flag to detect if a test altered @@ -367,49 +387,61 @@ def _runtest_inner( if ns.pgo: display_failure = False + test_name = result.test_name try: clear_caches() support.gc_collect() with save_env(ns, test_name): - refleak = _runtest_inner2(ns, test_name) + _load_run_test(result, ns) except support.ResourceDenied as msg: if not ns.quiet and not ns.pgo: print(f"{test_name} skipped -- {msg}", flush=True) - return ResourceDenied(test_name) + result.state = State.RESOURCE_DENIED + return except unittest.SkipTest as msg: if not ns.quiet and not ns.pgo: print(f"{test_name} skipped -- {msg}", flush=True) - return Skipped(test_name) + result.state = State.SKIPPED + return except support.TestFailedWithDetails as exc: msg = f"test {test_name} failed" if display_failure: msg = f"{msg} -- {exc}" print(msg, file=sys.stderr, flush=True) - return Failed(test_name, errors=exc.errors, failures=exc.failures) + result.state = State.FAILED + result.errors = exc.errors + result.failures = exc.failures + result.stats = exc.stats + return except support.TestFailed as exc: msg = f"test {test_name} failed" if display_failure: msg = f"{msg} -- {exc}" print(msg, file=sys.stderr, flush=True) - return Failed(test_name) + result.state = State.FAILED + result.stats = exc.stats + return except support.TestDidNotRun: - return DidNotRun(test_name) + result.state = State.DID_NOT_RUN + return except KeyboardInterrupt: print() - return Interrupted(test_name) + result.state = State.INTERRUPTED + return except: if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) - return UncaughtException(test_name) + result.state = State.UNCAUGHT_EXC + return - if refleak: - return RefLeak(test_name) if support.environment_altered: - return EnvChanged(test_name) - return Passed(test_name) + result.set_env_changed() + # Don't override the state if it was already set (REFLEAK or ENV_CHANGED) + if result.state is None: + result.state = State.PASSED def cleanup_test_droppings(test_name: str, verbose: int) -> None: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 62e6c6df36518c..fb1f80b0c054e3 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,3 +1,4 @@ +import dataclasses import faulthandler import json import os.path @@ -13,12 +14,13 @@ from test import support from test.support import os_helper +from test.support import TestStats from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - runtest, is_failed, TestResult, Interrupted, Timeout, ChildError, - PROGRESS_MIN_TIME, Passed, EnvChanged) + runtest, TestResult, State, + PROGRESS_MIN_TIME) from test.libregrtest.setup import setup_tests from test.libregrtest.utils import format_duration, print_warning @@ -43,9 +45,9 @@ def must_stop(result: TestResult, ns: Namespace) -> bool: - if isinstance(result, Interrupted): + if result.state == State.INTERRUPTED: return True - if ns.failfast and is_failed(result, ns): + if ns.failfast and result.is_failed(ns.fail_env_changed): return True return False @@ -130,8 +132,8 @@ def stop(self): class MultiprocessResult(NamedTuple): result: TestResult # bpo-45410: stderr is written into stdout to keep messages order - stdout: str - error_msg: str + worker_stdout: str | None = None + err_msg: str | None = None ExcStr = str @@ -209,15 +211,12 @@ def stop(self) -> None: def mp_result_error( self, test_result: TestResult, - stdout: str = '', + stdout: str | None = None, err_msg=None ) -> MultiprocessResult: - test_result.duration_sec = time.monotonic() - self.start_time return MultiprocessResult(test_result, stdout, err_msg) def _run_process(self, test_name: str, tmp_dir: str, stdout_fh: TextIO) -> int: - self.start_time = time.monotonic() - self.current_test_name = test_name try: popen = run_test_in_subprocess(test_name, self.ns, tmp_dir, stdout_fh) @@ -306,38 +305,41 @@ def _runtest(self, test_name: str) -> MultiprocessResult: # gh-101634: Catch UnicodeDecodeError if stdout cannot be # decoded from encoding err_msg = f"Cannot read process stdout: {exc}" - return self.mp_result_error(ChildError(test_name), '', err_msg) + result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR) + return self.mp_result_error(result, err_msg=err_msg) if retcode is None: - return self.mp_result_error(Timeout(test_name), stdout) + result = TestResult(test_name, state=State.TIMEOUT) + return self.mp_result_error(result, stdout) err_msg = None if retcode != 0: err_msg = "Exit code %s" % retcode else: - stdout, _, result = stdout.rpartition("\n") + stdout, _, worker_json = stdout.rpartition("\n") stdout = stdout.rstrip() - if not result: + if not worker_json: err_msg = "Failed to parse worker stdout" else: try: # deserialize run_tests_worker() output - result = json.loads(result, object_hook=decode_test_result) + result = json.loads(worker_json, + object_hook=decode_test_result) except Exception as exc: err_msg = "Failed to parse worker JSON: %s" % exc - if err_msg is not None: - return self.mp_result_error(ChildError(test_name), stdout, err_msg) + if err_msg: + result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR) + return self.mp_result_error(result, stdout, err_msg) if tmp_files: msg = (f'\n\n' f'Warning -- {test_name} leaked temporary files ' f'({len(tmp_files)}): {", ".join(sorted(tmp_files))}') stdout += msg - if isinstance(result, Passed): - result = EnvChanged.from_passed(result) + result.set_env_changed() - return MultiprocessResult(result, stdout, err_msg) + return MultiprocessResult(result, stdout) def run(self) -> None: while not self._stopped: @@ -347,7 +349,9 @@ def run(self) -> None: except StopIteration: break + self.start_time = time.monotonic() mp_result = self._runtest(test_name) + mp_result.result.duration = time.monotonic() - self.start_time self.output.put((False, mp_result)) if must_stop(mp_result.result, self.ns): @@ -473,11 +477,11 @@ def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result text = str(result) - if mp_result.error_msg is not None: - # CHILD_ERROR - text += ' (%s)' % mp_result.error_msg - elif (result.duration_sec >= PROGRESS_MIN_TIME and not self.ns.pgo): - text += ' (%s)' % format_duration(result.duration_sec) + if mp_result.err_msg: + # MULTIPROCESSING_ERROR + text += ' (%s)' % mp_result.err_msg + elif (result.duration >= PROGRESS_MIN_TIME and not self.ns.pgo): + text += ' (%s)' % format_duration(result.duration) running = get_running(self.workers) if running and not self.ns.pgo: text += ' -- running: %s' % ', '.join(running) @@ -489,7 +493,7 @@ def _process_result(self, item: QueueOutput) -> bool: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") - result = ChildError("") + result = TestResult("", state=State.MULTIPROCESSING_ERROR) self.regrtest.accumulate_result(result) return True @@ -498,8 +502,8 @@ def _process_result(self, item: QueueOutput) -> bool: self.regrtest.accumulate_result(mp_result.result) self.display_result(mp_result) - if mp_result.stdout: - print(mp_result.stdout, flush=True) + if mp_result.worker_stdout: + print(mp_result.worker_stdout, flush=True) if must_stop(mp_result.result, self.ns): return True @@ -541,32 +545,20 @@ class EncodeTestResult(json.JSONEncoder): def default(self, o: Any) -> dict[str, Any]: if isinstance(o, TestResult): - result = vars(o) + result = dataclasses.asdict(o) result["__test_result__"] = o.__class__.__name__ return result return super().default(o) -def decode_test_result(d: dict[str, Any]) -> TestResult | dict[str, Any]: +def decode_test_result(d: dict[str, Any]) -> TestResult | TestStats | dict[str, Any]: """Decode a TestResult (sub)class object from a JSON dict.""" if "__test_result__" not in d: return d - cls_name = d.pop("__test_result__") - for cls in get_all_test_result_classes(): - if cls.__name__ == cls_name: - return cls(**d) - - -def get_all_test_result_classes() -> set[type[TestResult]]: - prev_count = 0 - classes = {TestResult} - while len(classes) > prev_count: - prev_count = len(classes) - to_add = [] - for cls in classes: - to_add.extend(cls.__subclasses__()) - classes.update(to_add) - return classes + d.pop('__test_result__') + if d['stats'] is not None: + d['stats'] = TestStats(**d['stats']) + return TestResult(**d) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index c7801b767c590c..164fe9806b5f0d 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -23,7 +23,7 @@ class SkipTestEnvironment(Exception): class saved_test_environment: """Save bits of the test environment and restore them at block exit. - with saved_test_environment(testname, verbose, quiet): + with saved_test_environment(test_name, verbose, quiet): #stuff Unless quiet is True, a warning is printed to stderr if any of @@ -34,8 +34,8 @@ class saved_test_environment: items is also printed. """ - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname + def __init__(self, test_name, verbose=0, quiet=False, *, pgo=False): + self.test_name = test_name self.verbose = verbose self.quiet = quiet self.pgo = pgo @@ -323,7 +323,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): restore(original) if not self.quiet and not self.pgo: print_warning( - f"{name} was modified by {self.testname}\n" + f"{name} was modified by {self.test_name}\n" f" Before: {original}\n" f" After: {current} ") return False diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 16a5056a33aa12..f28a3a2632c1c5 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -4,6 +4,7 @@ raise ImportError('support must be imported from the test package') import contextlib +import dataclasses import functools import getpass import _opcode @@ -118,17 +119,20 @@ class Error(Exception): class TestFailed(Error): """Test failed.""" + def __init__(self, msg, *args, stats=None): + self.msg = msg + self.stats = stats + super().__init__(msg, *args) + + def __str__(self): + return self.msg class TestFailedWithDetails(TestFailed): """Test failed.""" - def __init__(self, msg, errors, failures): - self.msg = msg + def __init__(self, msg, errors, failures, stats): self.errors = errors self.failures = failures - super().__init__(msg, errors, failures) - - def __str__(self): - return self.msg + super().__init__(msg, errors, failures, stats=stats) class TestDidNotRun(Error): """Test did not run any subtests.""" @@ -1105,6 +1109,30 @@ def _filter_suite(suite, pred): newtests.append(test) suite._tests = newtests +@dataclasses.dataclass(slots=True) +class TestStats: + tests_run: int = 0 + failures: int = 0 + skipped: int = 0 + + @staticmethod + def from_unittest(result): + return TestStats(result.testsRun, + len(result.failures), + len(result.skipped)) + + @staticmethod + def from_doctest(results): + return TestStats(results.attempted, + results.failed, + results.skipped) + + def accumulate(self, stats): + self.tests_run += stats.tests_run + self.failures += stats.failures + self.skipped += stats.skipped + + def _run_suite(suite): """Run tests from a unittest.TestSuite-derived class.""" runner = get_test_runner(sys.stdout, @@ -1119,6 +1147,7 @@ def _run_suite(suite): if not result.testsRun and not result.skipped and not result.errors: raise TestDidNotRun if not result.wasSuccessful(): + stats = TestStats.from_unittest(result) if len(result.errors) == 1 and not result.failures: err = result.errors[0][1] elif len(result.failures) == 1 and not result.errors: @@ -1128,7 +1157,8 @@ def _run_suite(suite): if not verbose: err += "; run in verbose mode for details" errors = [(str(tc), exc_str) for tc, exc_str in result.errors] failures = [(str(tc), exc_str) for tc, exc_str in result.failures] - raise TestFailedWithDetails(err, errors, failures) + raise TestFailedWithDetails(err, errors, failures, stats=stats) + return result # By default, don't filter tests @@ -1237,7 +1267,7 @@ def run_unittest(*classes): else: suite.addTest(loader.loadTestsFromTestCase(cls)) _filter_suite(suite, match_test) - _run_suite(suite) + return _run_suite(suite) #======================================================================= # Check for the presence of docstrings. @@ -1277,13 +1307,18 @@ def run_doctest(module, verbosity=None, optionflags=0): else: verbosity = None - f, t = doctest.testmod(module, verbose=verbosity, optionflags=optionflags) - if f: - raise TestFailed("%d of %d doctests failed" % (f, t)) + results = doctest.testmod(module, + verbose=verbosity, + optionflags=optionflags) + if results.failed: + stats = TestStats.from_doctest(results) + raise TestFailed(f"{results.failed} of {results.attempted} " + f"doctests failed", + stats=stats) if verbose: print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, t)) - return f, t + (module.__name__, results.attempted)) + return results #======================================================================= diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 573d636de956d1..b38cb327f68ecc 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -309,7 +309,7 @@ def test_security(self): ('anonymous', '', 'pass')) def test_main(): - run_unittest(NetrcTestCase) + return run_unittest(NetrcTestCase) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_pep646_syntax.py b/Lib/test/test_pep646_syntax.py index 3ffa82dc55fa23..12a4227e4dc959 100644 --- a/Lib/test/test_pep646_syntax.py +++ b/Lib/test/test_pep646_syntax.py @@ -320,7 +320,7 @@ def test_main(verbose=False): from test import support from test import test_pep646_syntax - support.run_doctest(test_pep646_syntax, verbose) + return support.run_doctest(test_pep646_syntax, verbose) if __name__ == "__main__": test_main(verbose=True) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0c1400c8105037..1c02d802c0b061 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -19,7 +19,7 @@ import unittest from test import libregrtest from test import support -from test.support import os_helper +from test.support import os_helper, TestStats from test.libregrtest import utils, setup if not support.has_subprocess_support: @@ -409,7 +409,9 @@ def regex_search(self, regex, output): self.fail("%r not found in %r" % (regex, output)) return match - def check_line(self, output, regex): + def check_line(self, output, regex, full=False): + if full: + regex += '\n' regex = re.compile(r'^' + regex, re.MULTILINE) self.assertRegex(output, regex) @@ -421,21 +423,27 @@ def parse_executed_tests(self, output): def check_executed_tests(self, output, tests, skipped=(), failed=(), env_changed=(), omitted=(), - rerun={}, no_test_ran=(), + rerun={}, run_no_tests=(), + resource_denied=(), randomize=False, interrupted=False, - fail_env_changed=False): + fail_env_changed=False, + *, stats): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] + if isinstance(resource_denied, str): + resource_denied = [resource_denied] if isinstance(failed, str): failed = [failed] if isinstance(env_changed, str): env_changed = [env_changed] if isinstance(omitted, str): omitted = [omitted] - if isinstance(no_test_ran, str): - no_test_ran = [no_test_ran] + if isinstance(run_no_tests, str): + run_no_tests = [run_no_tests] + if isinstance(stats, int): + stats = TestStats(stats) executed = self.parse_executed_tests(output) if randomize: @@ -479,12 +487,12 @@ def list_regex(line_format, tests): regex = LOG_PREFIX + f"Re-running {name} in verbose mode \\(matching: {match}\\)" self.check_line(output, regex) - if no_test_ran: - regex = list_regex('%s test%s run no tests', no_test_ran) + if run_no_tests: + regex = list_regex('%s test%s run no tests', run_no_tests) self.check_line(output, regex) good = (len(tests) - len(skipped) - len(failed) - - len(omitted) - len(env_changed) - len(no_test_ran)) + - len(omitted) - len(env_changed) - len(run_no_tests)) if good: regex = r'%s test%s OK\.$' % (good, plural(good)) if not skipped and not failed and good > 1: @@ -494,6 +502,33 @@ def list_regex(line_format, tests): if interrupted: self.check_line(output, 'Test suite interrupted by signal SIGINT.') + # Total tests + parts = [f'run={stats.tests_run:,}'] + if stats.failures: + parts.append(f'failures={stats.failures:,}') + if stats.skipped: + parts.append(f'skipped={stats.skipped:,}') + line = fr'Total tests: {" ".join(parts)}' + self.check_line(output, line, full=True) + + # Total test files + report = [f'success={good}'] + if failed: + report.append(f'failed={len(failed)}') + if env_changed: + report.append(f'env_changed={len(env_changed)}') + if skipped: + report.append(f'skipped={len(skipped)}') + if resource_denied: + report.append(f'resource_denied={len(resource_denied)}') + if rerun: + report.append(f'rerun={len(rerun)}') + if run_no_tests: + report.append(f'run_no_tests={len(run_no_tests)}') + line = fr'Total test files: {" ".join(report)}' + self.check_line(output, line, full=True) + + # Result result = [] if failed: result.append('FAILURE') @@ -508,10 +543,8 @@ def list_regex(line_format, tests): result.append('SUCCESS') result = ', '.join(result) if rerun: - self.check_line(output, 'Tests result: FAILURE') result = 'FAILURE then %s' % result - - self.check_line(output, 'Tests result: %s' % result) + self.check_line(output, f'Result: {result}', full=True) def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) @@ -604,7 +637,8 @@ def setUp(self): def check_output(self, output): self.parse_random_seed(output) - self.check_executed_tests(output, self.tests, randomize=True) + self.check_executed_tests(output, self.tests, + randomize=True, stats=len(self.tests)) def run_tests(self, args): output = self.run_python(args) @@ -718,7 +752,8 @@ def test_failing(self): tests = [test_ok, test_failing] output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST) - self.check_executed_tests(output, tests, failed=test_failing) + self.check_executed_tests(output, tests, failed=test_failing, + stats=TestStats(2, 1)) def test_resources(self): # test -u command line option @@ -737,17 +772,21 @@ def test_pass(self): # -u all: 2 resources enabled output = self.run_tests('-u', 'all', *test_names) - self.check_executed_tests(output, test_names) + self.check_executed_tests(output, test_names, stats=2) # -u audio: 1 resource enabled output = self.run_tests('-uaudio', *test_names) self.check_executed_tests(output, test_names, - skipped=tests['network']) + skipped=tests['network'], + resource_denied=tests['network'], + stats=1) # no option: 0 resources enabled output = self.run_tests(*test_names) self.check_executed_tests(output, test_names, - skipped=test_names) + skipped=test_names, + resource_denied=test_names, + stats=0) def test_random(self): # test -r and --randseed command line option @@ -795,7 +834,8 @@ def test_fromfile(self): previous = name output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + stats = len(tests) + self.check_executed_tests(output, tests, stats=stats) # test format '[2/7] test_opcodes' with open(filename, "w") as fp: @@ -803,7 +843,7 @@ def test_fromfile(self): print("[%s/%s] %s" % (index, len(tests), name), file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) # test format 'test_opcodes' with open(filename, "w") as fp: @@ -811,7 +851,7 @@ def test_fromfile(self): print(name, file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) # test format 'Lib/test/test_opcodes.py' with open(filename, "w") as fp: @@ -819,20 +859,20 @@ def test_fromfile(self): print('Lib/test/%s.py' % name, file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, omitted=test, - interrupted=True) + interrupted=True, stats=0) def test_slowest(self): # test --slowest tests = [self.create_test() for index in range(3)] output = self.run_tests("--slowest", *tests) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=len(tests)) regex = ('10 slowest tests:\n' '(?:- %s: .*\n){%s}' % (self.TESTNAME_REGEX, len(tests))) @@ -851,7 +891,8 @@ def test_slowest_interrupted(self): args = ("--slowest", test) output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, - omitted=test, interrupted=True) + omitted=test, interrupted=True, + stats=0) regex = ('10 slowest tests:\n') self.check_line(output, regex) @@ -860,7 +901,7 @@ def test_coverage(self): # test --coverage test = self.create_test('coverage') output = self.run_tests("--coverage", test) - self.check_executed_tests(output, [test]) + self.check_executed_tests(output, [test], stats=1) regex = (r'lines +cov% +module +\(path\)\n' r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) @@ -890,7 +931,8 @@ def test_run(self): """) test = self.create_test('forever', code=code) output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST) - self.check_executed_tests(output, [test]*3, failed=test) + self.check_executed_tests(output, [test]*3, failed=test, + stats=TestStats(1, 1)) def check_leak(self, code, what): test = self.create_test('huntrleaks', code=code) @@ -900,7 +942,7 @@ def check_leak(self, code, what): output = self.run_tests('--huntrleaks', '6:3:', test, exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) - self.check_executed_tests(output, [test], failed=test) + self.check_executed_tests(output, [test], failed=test, stats=1) line = 'beginning 9 repetitions\n123456789\n.........\n' self.check_line(output, re.escape(line)) @@ -982,7 +1024,7 @@ def test_crashed(self): tests = [crash_test] output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, tests, failed=crash_test, - randomize=True) + randomize=True, stats=0) def parse_methods(self, output): regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) @@ -1077,13 +1119,14 @@ def test_env_changed(self): # don't fail by default output = self.run_tests(testname) - self.check_executed_tests(output, [testname], env_changed=testname) + self.check_executed_tests(output, [testname], + env_changed=testname, stats=1) # fail with --fail-env-changed output = self.run_tests("--fail-env-changed", testname, exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=testname, - fail_env_changed=True) + fail_env_changed=True, stats=1) def test_rerun_fail(self): # FAILURE then FAILURE @@ -1102,7 +1145,9 @@ def test_fail_always(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname, rerun={testname: "test_fail_always"}) + failed=testname, + rerun={testname: "test_fail_always"}, + stats=TestStats(1, 1)) def test_rerun_success(self): # FAILURE then SUCCESS @@ -1123,7 +1168,8 @@ def test_fail_once(self): output = self.run_tests("-w", testname, exitcode=0) self.check_executed_tests(output, [testname], - rerun={testname: "test_fail_once"}) + rerun={testname: "test_fail_once"}, + stats=1) def test_rerun_setup_class_hook_failure(self): # FAILURE then FAILURE @@ -1143,7 +1189,8 @@ def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "ExampleTests"}) + rerun={testname: "ExampleTests"}, + stats=0) def test_rerun_teardown_class_hook_failure(self): # FAILURE then FAILURE @@ -1163,7 +1210,8 @@ def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "ExampleTests"}) + rerun={testname: "ExampleTests"}, + stats=1) def test_rerun_setup_module_hook_failure(self): # FAILURE then FAILURE @@ -1182,7 +1230,8 @@ def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: testname}) + rerun={testname: testname}, + stats=0) def test_rerun_teardown_module_hook_failure(self): # FAILURE then FAILURE @@ -1201,7 +1250,8 @@ def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: testname}) + rerun={testname: testname}, + stats=1) def test_rerun_setup_hook_failure(self): # FAILURE then FAILURE @@ -1220,7 +1270,8 @@ def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}) + rerun={testname: "test_success"}, + stats=1) def test_rerun_teardown_hook_failure(self): # FAILURE then FAILURE @@ -1239,7 +1290,8 @@ def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}) + rerun={testname: "test_success"}, + stats=1) def test_rerun_async_setup_hook_failure(self): # FAILURE then FAILURE @@ -1258,7 +1310,8 @@ async def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}) + rerun={testname: "test_success"}, + stats=1) def test_rerun_async_teardown_hook_failure(self): # FAILURE then FAILURE @@ -1277,7 +1330,8 @@ async def test_success(self): output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}) + rerun={testname: "test_success"}, + stats=1) def test_no_tests_ran(self): code = textwrap.dedent(""" @@ -1291,7 +1345,9 @@ def test_bug(self): output = self.run_tests(testname, "-m", "nosuchtest", exitcode=EXITCODE_NO_TESTS_RAN) - self.check_executed_tests(output, [testname], no_test_ran=testname) + self.check_executed_tests(output, [testname], + run_no_tests=testname, + stats=0) def test_no_tests_ran_skip(self): code = textwrap.dedent(""" @@ -1304,7 +1360,8 @@ def test_skipped(self): testname = self.create_test(code=code) output = self.run_tests(testname) - self.check_executed_tests(output, [testname]) + self.check_executed_tests(output, [testname], + stats=TestStats(1, skipped=1)) def test_no_tests_ran_multiple_tests_nonexistent(self): code = textwrap.dedent(""" @@ -1320,7 +1377,8 @@ def test_bug(self): output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname, testname2], - no_test_ran=[testname, testname2]) + run_no_tests=[testname, testname2], + stats=0) def test_no_test_ran_some_test_exist_some_not(self): code = textwrap.dedent(""" @@ -1343,7 +1401,8 @@ def test_other_bug(self): output = self.run_tests(testname, testname2, "-m", "nosuchtest", "-m", "test_other_bug", exitcode=0) self.check_executed_tests(output, [testname, testname2], - no_test_ran=[testname]) + run_no_tests=[testname], + stats=1) @support.cpython_only def test_uncollectable(self): @@ -1370,7 +1429,8 @@ def test_garbage(self): exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], - fail_env_changed=True) + fail_env_changed=True, + stats=1) def test_multiprocessing_timeout(self): code = textwrap.dedent(r""" @@ -1396,7 +1456,7 @@ def test_sleep(self): output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname) + failed=testname, stats=0) self.assertRegex(output, re.compile('%s timed out' % testname, re.MULTILINE)) @@ -1430,7 +1490,8 @@ def test_unraisable_exc(self): exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], - fail_env_changed=True) + fail_env_changed=True, + stats=1) self.assertIn("Warning -- Unraisable exception", output) self.assertIn("Exception: weakref callback bug", output) @@ -1462,7 +1523,8 @@ def test_threading_excepthook(self): exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], - fail_env_changed=True) + fail_env_changed=True, + stats=1) self.assertIn("Warning -- Uncaught thread exception", output) self.assertIn("Exception: bug in thread", output) @@ -1503,7 +1565,8 @@ def test_print_warning(self): output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], - fail_env_changed=True) + fail_env_changed=True, + stats=1) self.assertRegex(output, regex) def test_unicode_guard_env(self): @@ -1550,7 +1613,8 @@ def test_leak_tmp_file(self): self.check_executed_tests(output, testnames, env_changed=testnames, fail_env_changed=True, - randomize=True) + randomize=True, + stats=len(testnames)) for testname in testnames: self.assertIn(f"Warning -- {testname} leaked temporary " f"files (1): mytmpfile", @@ -1589,7 +1653,47 @@ def test_mp_decode_error(self): exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], failed=[testname], - randomize=True) + randomize=True, + stats=0) + + def test_doctest(self): + code = textwrap.dedent(fr''' + import doctest + import sys + from test import support + + def my_function(): + """ + Pass: + + >>> 1 + 1 + 2 + + Failure: + + >>> 2 + 3 + 23 + >>> 1 + 1 + 11 + + Skipped test (ignored): + + >>> id(1.0) # doctest: +SKIP + 7948648 + """ + + def test_main(): + testmod = sys.modules[__name__] + return support.run_doctest(testmod) + ''') + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + randomize=True, + stats=TestStats(4, 2, 1)) class TestUtils(unittest.TestCase): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b9352cb865d027..da2828c3b53af0 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -4250,7 +4250,7 @@ def test_main(module=None): old_factories = None try: - support.run_unittest(*test_classes) + return support.run_unittest(*test_classes) finally: from xml.etree import ElementPath # Restore mapping and path cache From f5ddbeeab7df58d42b870061df13de6364b47d13 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 19:26:20 +0200 Subject: [PATCH 464/598] gh-108822: Add Changelog entry for regrtest statistics (#108821) --- .../next/Tests/2023-09-02-19-06-52.gh-issue-108822.arTbBI.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-02-19-06-52.gh-issue-108822.arTbBI.rst diff --git a/Misc/NEWS.d/next/Tests/2023-09-02-19-06-52.gh-issue-108822.arTbBI.rst b/Misc/NEWS.d/next/Tests/2023-09-02-19-06-52.gh-issue-108822.arTbBI.rst new file mode 100644 index 00000000000000..e1c6df2adcb0ae --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-02-19-06-52.gh-issue-108822.arTbBI.rst @@ -0,0 +1,4 @@ +``regrtest`` now computes statistics on all tests: successes, failures and +skipped. ``test_netrc``, ``test_pep646_syntax`` and ``test_xml_etree`` now +return results in their ``test_main()`` function. Patch by Victor Stinner +and Alex Waygood. From bac1e6d695a11cff270e663197f3cecd001fa556 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 19:54:39 +0200 Subject: [PATCH 465/598] gh-108765: multiprocessing.h includes (#108823) --- Modules/_multiprocessing/multiprocessing.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index 296e0abb29a0f5..099004b437828e 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -10,6 +10,10 @@ #include "pythread.h" #include "pycore_signal.h" // _PyOS_IsMainThread() +#ifndef MS_WINDOWS +# include // sysconf() +#endif + /* * Platform includes and definitions */ From 0e6d582b3b73a88e71cae04327b31a1ee203722c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 19:54:59 +0200 Subject: [PATCH 466/598] gh-63760: Don't declare gethostname() on Solaris (#108817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 2005, Solaris defines gethostname(). socketmodule.c no longer has to define gethostname() for Solaris. Oracle Solaris and OpenSolaris have patches to remove the gethostname() definition in Python: * https://github.com/oracle/solaris-userland/blob/master/components/python/python37/patches/15-gethostname.patch * https://github.com/OpenIndiana/oi-userland/blob/oi/hipster/components/python/python37/patches/15-gethostname.patch * https://github.com/omniosorg/omnios-build/blob/master/build/python27/patches/24-gethostname.patch Co-authored-by: Jakub Kulík --- .../next/Build/2023-09-02-18-04-15.gh-issue-63760.r8hJ6q.rst | 3 +++ Modules/socketmodule.c | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-09-02-18-04-15.gh-issue-63760.r8hJ6q.rst diff --git a/Misc/NEWS.d/next/Build/2023-09-02-18-04-15.gh-issue-63760.r8hJ6q.rst b/Misc/NEWS.d/next/Build/2023-09-02-18-04-15.gh-issue-63760.r8hJ6q.rst new file mode 100644 index 00000000000000..9a7249e923e0c7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-09-02-18-04-15.gh-issue-63760.r8hJ6q.rst @@ -0,0 +1,3 @@ +Fix Solaris build: no longer redefine the ``gethostname()`` function. Solaris +defines the function since 2005. Patch by Victor Stinner, original patch by +Jakub Kulík. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 74b1c1c661604f..90592ffc152fc1 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -111,11 +111,6 @@ Local naming conventions: #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState -// gethostname() prototype missing from Solaris standard header files -#ifdef __sun -extern int gethostname(char *, int); -#endif - #ifdef _Py_MEMORY_SANITIZER # include #endif From ecc61a6d76ea329e56f98c4af0f24a17ed3b2d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Sat, 2 Sep 2023 20:57:47 +0200 Subject: [PATCH 467/598] gh-108765: include in termios.c on Solaris (#108825) --- Modules/termios.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/termios.c b/Modules/termios.c index a0613837ef9795..c779a757e4fa9b 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -27,6 +27,9 @@ #include #include +#if defined(__sun) && defined(__SVR4) +# include // ioctl() +#endif /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, * MDTR, MRI, and MRTS (apparently used internally by some things From 6fafa6b919227cab06d0e3d7b20120e72d9b2bfd Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sat, 2 Sep 2023 21:13:00 +0000 Subject: [PATCH 468/598] gh-108374: Add --disable-gil option to PCbuild/build.bat (#108729) This adds a `--disable-gil` option to PCbuild/build.bat. For now, all this does is define the Py_NOGIL macro. --- PCbuild/build.bat | 3 +++ PCbuild/pyproject.props | 1 + 2 files changed, 4 insertions(+) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index d333ceabd2e53a..e61267b5852a8f 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -33,6 +33,7 @@ echo. -k Attempt to kill any running Pythons before building (usually done echo. automatically by the pythoncore project) echo. --pgo Build with Profile-Guided Optimization. This flag echo. overrides -c and -d +echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. echo. @@ -80,6 +81,7 @@ if "%~1"=="-q" (set verbose=/v:q /nologo /clp:summary) & shift & goto CheckOpts if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts @@ -172,6 +174,7 @@ echo on /p:IncludeExternals=%IncludeExternals%^ /p:IncludeCTypes=%IncludeCTypes%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ + /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index fd928cafd77aaa..9db400eebae388 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -40,6 +40,7 @@ $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + Py_NOGIL=1;%(PreprocessorDefinitions) MaxSpeed true From a52213bf830226fd969dc2a2ef8006c89edecc35 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 2 Sep 2023 23:15:54 +0200 Subject: [PATCH 469/598] gh-108765: pystrhex: Replace stdlib.h abs() with Py_ABS() (#108830) --- Python/pystrhex.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/pystrhex.c b/Python/pystrhex.c index ce456b79f1655f..38484f5a7d4227 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -3,7 +3,6 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex_with_sep() #include "pycore_unicodeobject.h" // _PyUnicode_CheckConsistency() -#include // abs() static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, PyObject* sep, int bytes_per_sep_group, @@ -44,7 +43,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, bytes_per_sep_group = 0; } - unsigned int abs_bytes_per_sep = abs(bytes_per_sep_group); + unsigned int abs_bytes_per_sep = Py_ABS(bytes_per_sep_group); Py_ssize_t resultlen = 0; if (bytes_per_sep_group && arglen > 0) { /* How many sep characters we'll be inserting. */ From f373c6b9483e12d7f6e03a631601149ed60ab883 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 2 Sep 2023 22:25:13 -0500 Subject: [PATCH 470/598] gh-107208: Fix iter_index() recipe to not swallow exceptions (gh-108835) gh-107208: iter_index now supports "stop" and no longer swallows ValueError --- Doc/library/itertools.rst | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index d0bb4696970198..42243715c2d93b 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -865,26 +865,22 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) - def iter_index(iterable, value, start=0): + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - try: - seq_index = iterable.index - except AttributeError: + seq_index = getattr(iterable, 'index', None) + if seq_index is None: # Slow path for general iterables - it = islice(iterable, start, None) - i = start - 1 - try: - while True: - yield (i := i + operator.indexOf(it, value) + 1) - except ValueError: - pass + it = islice(iterable, start, stop) + for i, element in enumerate(it, start): + if element is value or element == value: + yield i else: # Fast path for sequences i = start - 1 try: while True: - yield (i := seq_index(value, i+1)) + yield (i := seq_index(value, i+1, stop)) except ValueError: pass @@ -1331,6 +1327,21 @@ The following recipes have a more mathematical flavor: [] >>> list(iter_index(iter('AABCADEAF'), 'A', 10)) [] + >>> list(iter_index('AABCADEAF', 'A', 1, 7)) + [1, 4] + >>> list(iter_index(iter('AABCADEAF'), 'A', 1, 7)) + [1, 4] + >>> # Verify that ValueErrors not swallowed (gh-107208) + >>> def assert_no_value(iterable, forbidden_value): + ... for item in iterable: + ... if item == forbidden_value: + ... raise ValueError + ... yield item + ... + >>> list(iter_index(assert_no_value('AABCADEAF', 'B'), 'A')) + Traceback (most recent call last): + ... + ValueError >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] From 9c995abd780f3740dcd23ad85b78f2df5b4cdbaf Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Sun, 3 Sep 2023 11:48:47 +0300 Subject: [PATCH 471/598] gh-102837: improve test coverage for math module (#102523) - input checks for math_1(L989), math_1a(L1023), math_2(L1064,L1071), hypot(L2682), log(L2307), ldexp(L2168), ceil(L1165), floor(L1236,L1239) and dist(L2587,L2588,L2628). - drop inaccessible "if" branch (L3518) in perm_comb_small() - improve fsum coverage for exceptional cases (L1433,L1438,L1451,L1497), ditto fmod(L2378) - rewrite modf to fix inaccessible case(L2229), ditto for pow(L2988) (all line numbers are wrt the main branch at 5e6661bce9) --- Lib/test/test_math.py | 41 +++++++++++++++++++++++++++++++++++++++++ Modules/mathmodule.c | 17 +++++++---------- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 2bda61012164d1..b71d08b1124497 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -235,6 +235,10 @@ def __init__(self, value): def __index__(self): return self.value +class BadDescr: + def __get__(self, obj, objtype=None): + raise ValueError + class MathTests(unittest.TestCase): def ftest(self, name, got, expected, ulp_tol=5, abs_tol=0.0): @@ -324,6 +328,7 @@ def testAtan2(self): self.ftest('atan2(0, 1)', math.atan2(0, 1), 0) self.ftest('atan2(1, 1)', math.atan2(1, 1), math.pi/4) self.ftest('atan2(1, 0)', math.atan2(1, 0), math.pi/2) + self.ftest('atan2(1, -1)', math.atan2(1, -1), 3*math.pi/4) # math.atan2(0, x) self.ftest('atan2(0., -inf)', math.atan2(0., NINF), math.pi) @@ -417,16 +422,22 @@ def __ceil__(self): return 42 class TestNoCeil: pass + class TestBadCeil: + __ceil__ = BadDescr() self.assertEqual(math.ceil(TestCeil()), 42) self.assertEqual(math.ceil(FloatCeil()), 42) self.assertEqual(math.ceil(FloatLike(42.5)), 43) self.assertRaises(TypeError, math.ceil, TestNoCeil()) + self.assertRaises(ValueError, math.ceil, TestBadCeil()) t = TestNoCeil() t.__ceil__ = lambda *args: args self.assertRaises(TypeError, math.ceil, t) self.assertRaises(TypeError, math.ceil, t, 0) + self.assertEqual(math.ceil(FloatLike(+1.0)), +1.0) + self.assertEqual(math.ceil(FloatLike(-1.0)), -1.0) + @requires_IEEE_754 def testCopysign(self): self.assertEqual(math.copysign(1, 42), 1.0) @@ -567,16 +578,22 @@ def __floor__(self): return 42 class TestNoFloor: pass + class TestBadFloor: + __floor__ = BadDescr() self.assertEqual(math.floor(TestFloor()), 42) self.assertEqual(math.floor(FloatFloor()), 42) self.assertEqual(math.floor(FloatLike(41.9)), 41) self.assertRaises(TypeError, math.floor, TestNoFloor()) + self.assertRaises(ValueError, math.floor, TestBadFloor()) t = TestNoFloor() t.__floor__ = lambda *args: args self.assertRaises(TypeError, math.floor, t) self.assertRaises(TypeError, math.floor, t, 0) + self.assertEqual(math.floor(FloatLike(+1.0)), +1.0) + self.assertEqual(math.floor(FloatLike(-1.0)), -1.0) + def testFmod(self): self.assertRaises(TypeError, math.fmod) self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0) @@ -598,6 +615,7 @@ def testFmod(self): self.assertEqual(math.fmod(-3.0, NINF), -3.0) self.assertEqual(math.fmod(0.0, 3.0), 0.0) self.assertEqual(math.fmod(0.0, NINF), 0.0) + self.assertRaises(ValueError, math.fmod, INF, INF) def testFrexp(self): self.assertRaises(TypeError, math.frexp) @@ -714,6 +732,11 @@ def msum(iterable): s = msum(vals) self.assertEqual(msum(vals), math.fsum(vals)) + self.assertEqual(math.fsum([1.0, math.inf]), math.inf) + self.assertRaises(OverflowError, math.fsum, [1e+308, 1e+308]) + self.assertRaises(ValueError, math.fsum, [math.inf, -math.inf]) + self.assertRaises(TypeError, math.fsum, ['spam']) + def testGcd(self): gcd = math.gcd self.assertEqual(gcd(0, 0), 0) @@ -831,6 +854,8 @@ def testHypot(self): scale = FLOAT_MIN / 2.0 ** exp self.assertEqual(math.hypot(4*scale, 3*scale), 5*scale) + self.assertRaises(TypeError, math.hypot, *([1.0]*18), 'spam') + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "hypot() loses accuracy on machines with double rounding") @@ -966,6 +991,8 @@ class T(tuple): dist((1, 2, 3, 4), (5, 6, 7)) with self.assertRaises(ValueError): # Check dimension agree dist((1, 2, 3), (4, 5, 6, 7)) + with self.assertRaises(TypeError): + dist((1,)*17 + ("spam",), (1,)*18) with self.assertRaises(TypeError): # Rejects invalid types dist("abc", "xyz") int_too_big_for_float = 10 ** (sys.float_info.max_10_exp + 5) @@ -973,6 +1000,10 @@ class T(tuple): dist((1, int_too_big_for_float), (2, 3)) with self.assertRaises((ValueError, OverflowError)): dist((2, 3), (1, int_too_big_for_float)) + with self.assertRaises(TypeError): + dist((1,), 2) + with self.assertRaises(TypeError): + dist([1], 2) # Verify that the one dimensional case is equivalent to abs() for i in range(20): @@ -1111,6 +1142,7 @@ def test_lcm(self): def testLdexp(self): self.assertRaises(TypeError, math.ldexp) + self.assertRaises(TypeError, math.ldexp, 2.0, 1.1) self.ftest('ldexp(0,1)', math.ldexp(0,1), 0) self.ftest('ldexp(1,1)', math.ldexp(1,1), 2) self.ftest('ldexp(1,-1)', math.ldexp(1,-1), 0.5) @@ -1153,6 +1185,7 @@ def testLog(self): 2302.5850929940457) self.assertRaises(ValueError, math.log, -1.5) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) @@ -2378,6 +2411,14 @@ def __float__(self): # argument to a float. self.assertFalse(getattr(y, "converted", False)) + def test_input_exceptions(self): + self.assertRaises(TypeError, math.exp, "spam") + self.assertRaises(TypeError, math.erf, "spam") + self.assertRaises(TypeError, math.atan2, "spam", 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, "spam") + self.assertRaises(TypeError, math.atan2, 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + # Custom assertions. def assertIsNaN(self, value): diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index d929dcf65a7e32..2d896e7fe333a4 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2195,12 +2195,10 @@ math_modf_impl(PyObject *module, double x) double y; /* some platforms don't do the right thing for NaNs and infinities, so we take care of special cases directly. */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_INFINITY(x)) - return Py_BuildValue("(dd)", copysign(0., x), x); - else if (Py_IS_NAN(x)) - return Py_BuildValue("(dd)", x, x); - } + if (Py_IS_INFINITY(x)) + return Py_BuildValue("(dd)", copysign(0., x), x); + else if (Py_IS_NAN(x)) + return Py_BuildValue("(dd)", x, x); errno = 0; x = modf(x, &y); @@ -2950,7 +2948,8 @@ math_pow_impl(PyObject *module, double x, double y) else /* y < 0. */ r = odd_y ? copysign(0., x) : 0.; } - else if (Py_IS_INFINITY(y)) { + else { + assert(Py_IS_INFINITY(y)); if (fabs(x) == 1.0) r = 1.; else if (y > 0. && fabs(x) > 1.0) @@ -3480,9 +3479,7 @@ static const uint8_t factorial_trailing_zeros[] = { static PyObject * perm_comb_small(unsigned long long n, unsigned long long k, int iscomb) { - if (k == 0) { - return PyLong_FromLong(1); - } + assert(k != 0); /* For small enough n and k the result fits in the 64-bit range and can * be calculated without allocating intermediate PyLong objects. */ From 509bb61977cc8a4487efd3f9cdd63d9f7b86be62 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 3 Sep 2023 15:21:43 +0300 Subject: [PATCH 472/598] Reorder some test's decorators (GH-108804) For example, do not demand the 'cpu' resource if the test cannot be run due to non-working threads. --- Lib/test/test_io.py | 4 ++-- Lib/test/test_site.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 26ae40d93c84eb..6976a64bf77e7d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1541,8 +1541,8 @@ def test_read_all(self): self.assertEqual(b"abcdefg", bufio.read()) - @support.requires_resource('cpu') @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threads(self): try: # Write out many bytes with exactly the same number of 0's, @@ -1930,8 +1930,8 @@ def test_truncate_after_write(self): f.truncate() self.assertEqual(f.tell(), buffer_size + 2) - @support.requires_resource('cpu') @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threads(self): try: # Write out many bytes from many threads and test they were diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 9e701fd847acdf..20a96169e8be4e 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -465,10 +465,10 @@ def test_sitecustomize_executed(self): else: self.fail("sitecustomize not imported automatically") - @test.support.requires_resource('network') - @test.support.system_must_validate_cert @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') + @test.support.requires_resource('network') + @test.support.system_must_validate_cert def test_license_exists_at_url(self): # This test is a bit fragile since it depends on the format of the # string displayed by license in the absence of a LICENSE file. From 55846099b155833320bc6d64b03d902028bad439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D1=82=D0=B0=D0=BB=D0=B8=D0=B9=20=D0=94=D0=BC?= =?UTF-8?q?=D0=B8=D1=82=D1=80=D0=B8=D0=B5=D0=B2?= Date: Sun, 3 Sep 2023 16:35:13 +0300 Subject: [PATCH 473/598] Fix duplicated words 'Be be' (GH-108815) --- Misc/NEWS.d/3.10.0a3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 755109cbd376f4..70d5ce1319c99f 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -153,7 +153,7 @@ Allow an unparenthesized walrus in subscript indexes. .. section: Core and Builtins Make sure that the compiler front-end produces a well-formed control flow -graph. Be be more aggressive in the compiler back-end, as it is now safe to +graph. Be more aggressive in the compiler back-end, as it is now safe to do so. .. From 1796c191b43ed0787d83c07be7de8118fb10e8b0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 3 Sep 2023 17:28:14 +0300 Subject: [PATCH 474/598] gh-108494: Argument Clinic: inline parsing code for positional-only parameters in the limited C API (GH-108622) --- Modules/_io/clinic/bufferedio.c.h | 6 +- Modules/_io/clinic/bytesio.c.h | 3 +- Modules/_io/clinic/fileio.c.h | 3 +- Modules/_io/clinic/winconsoleio.c.h | 3 +- Modules/_multiprocessing/multiprocessing.c | 9 +- Modules/_posixsubprocess.c | 9 +- Modules/_struct.c | 12 +- Modules/clinic/_testclinic_limited.c.h | 22 +- Modules/overlapped.c | 16 +- Modules/resource.c | 9 +- PC/msvcrtmodule.c | 9 +- PC/winreg.c | 18 +- Tools/clinic/clinic.py | 743 ++++++++++++++------- 13 files changed, 555 insertions(+), 307 deletions(-) diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index b2c52cf2a95a44..7577bdec5c3b20 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -26,7 +26,6 @@ _io__BufferedIOBase_readinto(PyObject *self, PyObject *arg) Py_buffer buffer = {NULL, NULL}; if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } @@ -63,7 +62,6 @@ _io__BufferedIOBase_readinto1(PyObject *self, PyObject *arg) Py_buffer buffer = {NULL, NULL}; if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto1", "argument", "read-write bytes-like object", arg); goto exit; } @@ -590,7 +588,6 @@ _io__Buffered_readinto(buffered *self, PyObject *arg) Py_buffer buffer = {NULL, NULL}; if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } @@ -627,7 +624,6 @@ _io__Buffered_readinto1(buffered *self, PyObject *arg) Py_buffer buffer = {NULL, NULL}; if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto1", "argument", "read-write bytes-like object", arg); goto exit; } @@ -1098,4 +1094,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=9e09091995ae02b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f940cea085f0bf91 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index d7364779200827..d42ab48cef2859 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -328,7 +328,6 @@ _io_BytesIO_readinto(bytesio *self, PyObject *arg) Py_buffer buffer = {NULL, NULL}; if (PyObject_GetBuffer(arg, &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } @@ -538,4 +537,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8ab65edc03edbfe0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b753fdf1ba36c461 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 29f8cf6aa9a85c..deb99fa9d99bd0 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -245,7 +245,6 @@ _io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s goto exit; } if (PyObject_GetBuffer(args[0], &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } @@ -536,4 +535,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=238dd48819076434 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2ce6ce923ccef86e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 0683eecdfebb37..ecc71e552c23f4 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -243,7 +243,6 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject * goto exit; } if (PyObject_GetBuffer(args[0], &buffer, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } @@ -465,4 +464,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=7be51d48ddb7c8c8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=37febc4c96732b3b input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 16b5cb5dd9ec7a..2e6d8eb68c0243 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -14,16 +14,17 @@ class HANDLE_converter(CConverter): type = "HANDLE" format_unit = '"F_HANDLE"' - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" {paramname} = PyLong_AsVoidPtr({argname}); if (!{paramname} && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3e537d244034affb]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=3cf0318efc6a8772]*/ /*[clinic input] module _multiprocessing diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index ef76d26282e1b3..2898eedc3e3a8f 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -87,15 +87,16 @@ class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" {paramname} = PyLong_AsPid({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=5af1c116d56cbb5a]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=c94349aa1aad151d]*/ #include "clinic/_posixsubprocess.c.h" diff --git a/Modules/_struct.c b/Modules/_struct.c index 4ae21cce74f609..be4c23af384bb9 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -110,18 +110,20 @@ class cache_struct_converter(CConverter): c_default = "NULL" broken_limited_capi = True - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + assert not limited_capi + return self.format_code(""" if (!{converter}(module, {argname}, &{paramname})) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name, - converter=self.converter) + """, + argname=argname, + converter=self.converter) def cleanup(self): return "Py_XDECREF(%s);\n" % self.name [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=14e83804f599ed8f]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=c33b27d6b06006c6]*/ static int cache_struct_converter(PyObject *, PyObject *, PyStructObject **); diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index 93e7d7f98769cc..eff72cee24975c 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -37,7 +37,8 @@ my_int_func(PyObject *module, PyObject *arg_) int arg; int _return_value; - if (!PyArg_Parse(arg_, "i:my_int_func", &arg)) { + arg = PyLong_AsInt(arg_); + if (arg == -1 && PyErr_Occurred()) { goto exit; } _return_value = my_int_func_impl(module, arg); @@ -56,22 +57,31 @@ PyDoc_STRVAR(my_int_sum__doc__, "\n"); #define MY_INT_SUM_METHODDEF \ - {"my_int_sum", (PyCFunction)my_int_sum, METH_VARARGS, my_int_sum__doc__}, + {"my_int_sum", (PyCFunction)(void(*)(void))my_int_sum, METH_FASTCALL, my_int_sum__doc__}, static int my_int_sum_impl(PyObject *module, int x, int y); static PyObject * -my_int_sum(PyObject *module, PyObject *args) +my_int_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int x; int y; int _return_value; - if (!PyArg_ParseTuple(args, "ii:my_int_sum", - &x, &y)) + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_int_sum expected 2 arguments, got %zd", nargs); goto exit; + } + x = PyLong_AsInt(args[0]); + if (x == -1 && PyErr_Occurred()) { + goto exit; + } + y = PyLong_AsInt(args[1]); + if (y == -1 && PyErr_Occurred()) { + goto exit; + } _return_value = my_int_sum_impl(module, x, y); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -81,4 +91,4 @@ my_int_sum(PyObject *module, PyObject *args) exit: return return_value; } -/*[clinic end generated code: output=dcd5203d0d29df3a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5cf64baf978d2288 input=a9049054013a1b77]*/ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index c682e6adccc765..e23db22dadb18b 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -38,13 +38,14 @@ class pointer_converter(CConverter): format_unit = '"F_POINTER"' - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" {paramname} = PyLong_AsVoidPtr({argname}); if (!{paramname} && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) class OVERLAPPED_converter(pointer_converter): type = 'OVERLAPPED *' @@ -55,13 +56,14 @@ class HANDLE_converter(pointer_converter): class ULONG_PTR_converter(pointer_converter): type = 'ULONG_PTR' - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" {paramname} = (uintptr_t)PyLong_AsVoidPtr({argname}); if (!{paramname} && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) class DWORD_converter(unsigned_long_converter): type = 'DWORD' @@ -69,7 +71,7 @@ class DWORD_converter(unsigned_long_converter): class BOOL_converter(int_converter): type = 'BOOL' [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=8a07ea3018f4cec8]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=436f4440630a304c]*/ /*[clinic input] module _overlapped diff --git a/Modules/resource.c b/Modules/resource.c index 9e302a3a1ed962..d65509ec3436a2 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -23,15 +23,16 @@ class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" {paramname} = PyLong_AsPid({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=5af1c116d56cbb5a]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=c94349aa1aad151d]*/ #include "clinic/resource.c.h" diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index afc810adcf7499..d7c21766d709a6 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -38,13 +38,14 @@ class HANDLE_converter(CConverter): type = 'void *' format_unit = '"_Py_PARSE_UINTPTR"' - def parse_arg(self, argname, displayname): - return """ + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" {paramname} = PyLong_AsVoidPtr({argname}); if (!{paramname} && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) class HANDLE_return_converter(CReturnConverter): type = 'void *' @@ -74,7 +75,7 @@ class wchar_t_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = PyUnicode_FromOrdinal(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=1e8e9fa3538ec08f]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ff031be44ab3250d]*/ /*[clinic input] module msvcrt diff --git a/PC/winreg.c b/PC/winreg.c index 767127d34cab1b..77b80217ac0ab1 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -222,13 +222,15 @@ class HKEY_converter(CConverter): converter = 'clinic_HKEY_converter' broken_limited_capi = True - def parse_arg(self, argname, displayname): - return """ - if (!{converter}(_PyModule_GetState(module), {argname}, &{paramname})) {{{{ - goto exit; - }}}} - """.format(argname=argname, paramname=self.parser_name, - converter=self.converter) + def parse_arg(self, argname, displayname, *, limited_capi): + assert not limited_capi + return self.format_code(""" + if (!{converter}(_PyModule_GetState(module), {argname}, &{paramname})) {{{{ + goto exit; + }}}} + """, + argname=argname, + converter=self.converter) class HKEY_return_converter(CReturnConverter): type = 'HKEY' @@ -250,7 +252,7 @@ class self_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = (PyObject *)_return_value;\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=f8cb7034338aeaba]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=4979f33998ffb6f8]*/ #include "clinic/winreg.c.h" diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index b744ca7e5c8145..eca4747a3e4d57 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -990,6 +990,7 @@ def deprecate_keyword_use( params: dict[int, Parameter], argname_fmt: str | None, *, + fastcall: bool, limited_capi: bool, clinic: Clinic, ) -> str: @@ -1003,26 +1004,30 @@ def deprecate_keyword_use( if p.is_optional(): if argname_fmt: conditions.append(f"nargs < {i+1} && {argname_fmt % i}") - elif func.kind.new_or_init or limited_capi: - conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") - containscheck = "PyDict_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: + elif fastcall: conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") containscheck = "PySequence_Contains" clinic.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") + containscheck = "PyDict_Contains" + clinic.add_include('pycore_runtime.h', '_Py_ID()') else: conditions = [f"nargs < {i+1}"] condition = ") || (".join(conditions) if len(conditions) > 1: condition = f"(({condition}))" if last_param.is_optional(): - if limited_capi: - condition = f"kwargs && PyDict_Size(kwargs) && {condition}" - elif func.kind.new_or_init: - condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" + if fastcall: + if limited_capi: + condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" + else: + condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" else: - condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" + if limited_capi: + condition = f"kwargs && PyDict_Size(kwargs) && {condition}" + else: + condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" names = [repr(p.name) for p in params.values()] pstr = pprint_words(names) pl = 's' if len(params) != 1 else '' @@ -1193,6 +1198,7 @@ def parser_body( add(field) return linear_format(output(), parser_declarations=declarations) + fastcall = not new_or_init limited_capi = clinic.limited_capi if limited_capi and (requires_defining_class or pseudo_args or (any(p.is_optional() for p in parameters) and @@ -1210,7 +1216,7 @@ def parser_body( parser_prototype = self.PARSER_PROTOTYPE_NOARGS parser_code = [] else: - assert not new_or_init + assert fastcall flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS @@ -1261,11 +1267,8 @@ def parser_body( {c_basename}({self_type}{self_name}, PyObject *%s) """ % argname) - if limited_capi: - parsearg = None - else: - displayname = parameters[0].get_displayname(0) - parsearg = converters[0].parse_arg(argname, displayname) + displayname = parameters[0].get_displayname(0) + parsearg = converters[0].parse_arg(argname, displayname, limited_capi=limited_capi) if parsearg is None: parsearg = """ if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ @@ -1284,27 +1287,8 @@ def parser_body( parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') - elif (not requires_defining_class and pos_only == len(parameters) and - not pseudo_args and limited_capi): - # positional-only for the limited C API - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - if all(isinstance(c, object_converter) and c.format_unit == 'O' for c in converters): - parser_code = [normalize_snippet(""" - if (!PyArg_UnpackTuple(args, "{name}", %d, %d, - {parse_arguments})) - goto exit; - """ % (min_pos, max_pos), indent=4)] - else: - parser_code = [normalize_snippet(""" - if (!PyArg_ParseTuple(args, "{format_units}:{name}", - {parse_arguments})) - goto exit; - """, indent=4)] - parser_definition = parser_body(parser_prototype, *parser_code) - elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: - if not new_or_init: + if fastcall: # positional-only, but no option groups # we only need one call to _PyArg_ParseStack @@ -1318,24 +1302,59 @@ def parser_body( flags = "METH_VARARGS" parser_prototype = self.PARSER_PROTOTYPE_VARARGS - nargs = 'PyTuple_GET_SIZE(args)' - argname_fmt = 'PyTuple_GET_ITEM(args, %d)' + if limited_capi: + nargs = 'PyTuple_Size(args)' + argname_fmt = 'PyTuple_GetItem(args, %d)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + argname_fmt = 'PyTuple_GET_ITEM(args, %d)' left_args = f"{nargs} - {max_pos}" max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos - parser_code = [normalize_snippet(""" - if (!_PyArg_CheckPositional("{name}", %s, %d, %s)) {{ - goto exit; - }} - """ % (nargs, min_pos, max_args), indent=4)] + if limited_capi: + parser_code = [] + if nargs != 'nargs': + parser_code.append(normalize_snippet(f'Py_ssize_t nargs = {nargs};', indent=4)) + nargs = 'nargs' + if min_pos == max_args: + pl = '' if min_pos == 1 else 's' + parser_code.append(normalize_snippet(f""" + if ({nargs} != {min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + if min_pos: + pl = '' if min_pos == 1 else 's' + parser_code.append(normalize_snippet(f""" + if ({nargs} < {min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + if max_args != NO_VARARG: + pl = '' if max_args == 1 else 's' + parser_code.append(normalize_snippet(f""" + if ({nargs} > {max_args}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + parser_code = [normalize_snippet(f""" + if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ + goto exit; + }}}} + """, indent=4)] has_optional = False for i, p in enumerate(parameters): - displayname = p.get_displayname(i+1) - argname = argname_fmt % i - if p.is_vararg(): - if not new_or_init: + if fastcall: parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); if (!%s) {{ @@ -1361,7 +1380,9 @@ def parser_body( ), indent=4)) continue - parsearg = p.converter.parse_arg(argname, displayname) + displayname = p.get_displayname(i+1) + argname = argname_fmt % i + parsearg = p.converter.parse_arg(argname, displayname, limited_capi=limited_capi) if parsearg is None: parser_code = None break @@ -1378,7 +1399,9 @@ def parser_body( if has_optional: parser_code.append("skip_optional:") else: - if not new_or_init: + if limited_capi: + fastcall = False + if fastcall: parser_code = [normalize_snippet(""" if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", {parse_arguments})) {{ @@ -1386,6 +1409,8 @@ def parser_body( }} """, indent=4)] else: + flags = "METH_VARARGS" + parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_code = [normalize_snippet(""" if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ @@ -1421,25 +1446,10 @@ def parser_body( nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if limited_capi: - # positional-or-keyword arguments - flags = "METH_VARARGS|METH_KEYWORDS" - - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - parser_code = [normalize_snippet(""" - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, - {parse_arguments})) - goto exit; - """, indent=4)] - declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, None, - limited_capi=limited_capi, - clinic=clinic) - parser_code.append(code) + parser_code = None + fastcall = False - elif not new_or_init: + elif fastcall: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS argname_fmt = 'args[%d]' @@ -1477,11 +1487,12 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - if not limited_capi: + if parser_code is not None: if deprecated_keywords: code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, - limited_capi=limited_capi, - clinic=clinic) + clinic=clinic, + fastcall=fastcall, + limited_capi=limited_capi) parser_code.append(code) add_label: str | None = None @@ -1490,7 +1501,7 @@ def parser_body( raise ValueError("defining_class should be the first " "parameter (after self)") displayname = p.get_displayname(i+1) - parsearg = p.converter.parse_arg(argname_fmt % i, displayname) + parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=limited_capi) if parsearg is None: parser_code = None break @@ -1542,40 +1553,56 @@ def parser_body( }} """ % add_label, indent=4)) - if parser_code is not None: - if add_label: - parser_code.append("%s:" % add_label) + if parser_code is not None: + if add_label: + parser_code.append("%s:" % add_label) + else: + declarations = declare_parser(f, clinic=clinic, + hasformat=True, + limited_capi=limited_capi) + if limited_capi: + # positional-or-keyword arguments + assert not fastcall + flags = "METH_VARARGS|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + parser_code = [normalize_snippet(""" + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, + {parse_arguments})) + goto exit; + """, indent=4)] + declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" + + elif fastcall: + parser_code = [normalize_snippet(""" + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] else: - declarations = declare_parser(f, clinic=clinic, - hasformat=True, - limited_capi=clinic.limited_capi) - if not new_or_init: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, None, - limited_capi=limited_capi, - clinic=clinic) - parser_code.append(code) + parser_code = [normalize_snippet(""" + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, None, + clinic=clinic, + fastcall=fastcall, + limited_capi=limited_capi) + parser_code.append(code) if deprecated_positionals: code = self.deprecate_positional_use(f, deprecated_positionals) # Insert the deprecation code before parameter parsing. parser_code.insert(0, code) + assert parser_prototype is not None parser_definition = parser_body(parser_prototype, *parser_code, declarations=declarations) @@ -1619,6 +1646,9 @@ def parser_body( if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): methoddef_cast = "(PyCFunction)" methoddef_cast_end = "" + elif limited_capi: + methoddef_cast = "(PyCFunction)(void(*)(void))" + methoddef_cast_end = "" else: methoddef_cast = "_PyCFunction_CAST(" methoddef_cast_end = ")" @@ -3030,11 +3060,11 @@ def copy( def get_displayname(self, i: int) -> str: if i == 0: - return '"argument"' + return 'argument' if not self.is_positional_only(): - return f'"argument {self.name!r}"' + return f'argument {self.name!r}' else: - return f'"argument {i}"' + return f'argument {i}' def render_docstring(self) -> str: add, out = text_accumulator() @@ -3458,41 +3488,78 @@ def pre_render(self) -> None: """ pass - def parse_arg(self, argname: str, displayname: str) -> str | None: + def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: + assert '"' not in expected + if limited_capi: + if expected_literal: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be {expected}, not %.50s", ' + f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') + else: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be %.50s, not %.50s", ' + f'"{expected}", ' + f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') + else: + if expected_literal: + expected = f'"{expected}"' + return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' + + def format_code(self, fmt: str, *, + argname: str, + bad_argument: str | None = None, + bad_argument2: str | None = None, + **kwargs: Any) -> str: + if '{bad_argument}' in fmt: + if not bad_argument: + raise TypeError("required 'bad_argument' argument") + fmt = fmt.replace('{bad_argument}', bad_argument) + if '{bad_argument2}' in fmt: + if not bad_argument2: + raise TypeError("required 'bad_argument2' argument") + fmt = fmt.replace('{bad_argument2}', bad_argument2) + return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'O&': - return """ + return self.format_code(""" if (!{converter}({argname}, &{paramname})) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name, - converter=self.converter) + """, + argname=argname, + converter=self.converter) if self.format_unit == 'O!': cast = '(%s)' % self.type if self.type != 'PyObject *' else '' if self.subclass_of in type_checks: typecheck, typename = type_checks[self.subclass_of] - return """ + return self.format_code(""" if (!{typecheck}({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "{typename}", {argname}); + {bad_argument} goto exit; }}}} {paramname} = {cast}{argname}; - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname, typecheck=typecheck, - typename=typename, cast=cast) - return """ + """, + argname=argname, + bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), + typecheck=typecheck, typename=typename, cast=cast) + return self.format_code(""" if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, ({subclass_of})->tp_name, {argname}); + {bad_argument} goto exit; }}}} {paramname} = {cast}{argname}; - """.format(argname=argname, paramname=self.parser_name, - subclass_of=self.subclass_of, cast=cast, - displayname=displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', + expected_literal=False, limited_capi=limited_capi), + subclass_of=self.subclass_of, cast=cast) if self.format_unit == 'O': cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - return """ + return self.format_code(""" {paramname} = {cast}{argname}; - """.format(argname=argname, paramname=self.parser_name, cast=cast) + """, + argname=argname, cast=cast) return None def set_template_dict(self, template_dict: TemplateDict) -> None: @@ -3567,22 +3634,24 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: self.default = bool(self.default) self.c_default = str(int(self.default)) - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'i': - return """ + return self.format_code(""" {paramname} = PyLong_AsInt({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) elif self.format_unit == 'p': - return """ + return self.format_code(""" {paramname} = PyObject_IsTrue({argname}); if ({paramname} < 0) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class defining_class_converter(CConverter): """ @@ -3618,9 +3687,9 @@ def converter_init(self) -> None: if self.c_default == '"\'"': self.c_default = r"'\''" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'c': - return """ + return self.format_code(""" if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ {paramname} = PyBytes_AS_STRING({argname})[0]; }}}} @@ -3628,12 +3697,14 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: {paramname} = PyByteArray_AS_STRING({argname})[0]; }}}} else {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "a byte string of length 1", {argname}); + {bad_argument} goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) @add_legacy_c_converter('B', bitwise=True) @@ -3647,9 +3718,9 @@ def converter_init(self, *, bitwise: bool = False) -> None: if bitwise: self.format_unit = 'B' - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'b': - return """ + return self.format_code(""" {{{{ long ival = PyLong_AsLong({argname}); if (ival == -1 && PyErr_Occurred()) {{{{ @@ -3669,9 +3740,10 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: {paramname} = (unsigned char) ival; }}}} }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) elif self.format_unit == 'B': - return """ + return self.format_code(""" {{{{ unsigned long ival = PyLong_AsUnsignedLongMask({argname}); if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ @@ -3681,8 +3753,9 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: {paramname} = (unsigned char) ival; }}}} }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class byte_converter(unsigned_char_converter): pass @@ -3692,9 +3765,9 @@ class short_converter(CConverter): format_unit = 'h' c_ignored_default = "0" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'h': - return """ + return self.format_code(""" {{{{ long ival = PyLong_AsLong({argname}); if (ival == -1 && PyErr_Occurred()) {{{{ @@ -3714,8 +3787,9 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: {paramname} = (short) ival; }}}} }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class unsigned_short_converter(CConverter): type = 'unsigned short' @@ -3730,15 +3804,33 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.add_include('pycore_long.h', '_PyLong_UnsignedShort_Converter()') - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'H': - return """ + return self.format_code(""" {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > USHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned short"); + goto exit; + }}}} + {paramname} = (unsigned short) uval; + }}}} + """, + argname=argname) @add_legacy_c_converter('C', accept={str}) class int_converter(CConverter): @@ -3757,28 +3849,31 @@ def converter_init( if type is not None: self.type = type - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'i': - return """ + return self.format_code(""" {paramname} = PyLong_AsInt({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) + """, + argname=argname) elif self.format_unit == 'C': - return """ + return self.format_code(""" if (!PyUnicode_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "a unicode character", {argname}); + {bad_argument} goto exit; }}}} if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "a unicode character", {argname}); + {bad_argument} goto exit; }}}} {paramname} = PyUnicode_READ_CHAR({argname}, 0); - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class unsigned_int_converter(CConverter): type = 'unsigned int' @@ -3793,15 +3888,33 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.add_include('pycore_long.h', '_PyLong_UnsignedInt_Converter()') - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'I': - return """ + return self.format_code(""" {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > UINT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned int"); + goto exit; + }}}} + {paramname} = (unsigned int) uval; + }}}} + """, + argname=argname) class long_converter(CConverter): type = 'long' @@ -3809,15 +3922,16 @@ class long_converter(CConverter): format_unit = 'l' c_ignored_default = "0" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'l': - return """ + return self.format_code(""" {paramname} = PyLong_AsLong({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class unsigned_long_converter(CConverter): type = 'unsigned long' @@ -3832,17 +3946,28 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.add_include('pycore_long.h', '_PyLong_UnsignedLong_Converter()') - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'k': - return """ + return self.format_code(""" if (!PyLong_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "int", {argname}); + {bad_argument} goto exit; }}}} {paramname} = PyLong_AsUnsignedLongMask({argname}); - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLong({argname}); + if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) class long_long_converter(CConverter): type = 'long long' @@ -3850,15 +3975,16 @@ class long_long_converter(CConverter): format_unit = 'L' c_ignored_default = "0" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'L': - return """ + return self.format_code(""" {paramname} = PyLong_AsLongLong({argname}); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class unsigned_long_long_converter(CConverter): type = 'unsigned long long' @@ -3873,17 +3999,28 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.add_include('pycore_long.h', '_PyLong_UnsignedLongLong_Converter()') - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'K': - return """ + return self.format_code(""" if (!PyLong_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "int", {argname}); + {bad_argument} goto exit; }}}} {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLongLong({argname}); + if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) class Py_ssize_t_converter(CConverter): type = 'Py_ssize_t' @@ -3901,12 +4038,16 @@ def converter_init(self, *, accept: TypeSet = {int}) -> None: else: fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'n': - return """ + if limited_capi: + PyNumber_Index = 'PyNumber_Index' + else: + PyNumber_Index = '_PyNumber_Index' + return self.format_code(""" {{{{ Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index({argname}); + PyObject *iobj = {PyNumber_Index}({argname}); if (iobj != NULL) {{{{ ival = PyLong_AsSsize_t(iobj); Py_DECREF(iobj); @@ -3916,8 +4057,28 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: }}}} {paramname} = ival; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname, + PyNumber_Index=PyNumber_Index) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + if ({argname} != Py_None) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), + ) class slice_index_converter(CConverter): @@ -3926,11 +4087,51 @@ class slice_index_converter(CConverter): def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: if accept == {int}: self.converter = '_PyEval_SliceIndexNotNone' + self.nullable = False elif accept == {int, NoneType}: self.converter = '_PyEval_SliceIndex' + self.nullable = True else: fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + if self.nullable: + return self.format_code(""" + if (!Py_IsNone({argname})) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + return 0; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "None or have an __index__ method"); + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + goto exit; + }}}} + """, + argname=argname) + class size_t_converter(CConverter): type = 'size_t' converter = '_PyLong_Size_t_Converter' @@ -3940,15 +4141,25 @@ def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: self.add_include('pycore_long.h', '_PyLong_Size_t_Converter()') - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'n': - return """ + return self.format_code(""" {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsSize_t({argname}); + if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) class fildes_converter(CConverter): @@ -3960,12 +4171,13 @@ def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: '_PyLong_FileDescriptor_Converter()') def _parse_arg(self, argname: str, displayname: str) -> str | None: - return """ + return self.format_code(""" {paramname} = PyObject_AsFileDescriptor({argname}); if ({paramname} == -1) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name) + """, + argname=argname) class float_converter(CConverter): @@ -3974,9 +4186,9 @@ class float_converter(CConverter): format_unit = 'f' c_ignored_default = "0.0" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'f': - return """ + return self.format_code(""" if (PyFloat_CheckExact({argname})) {{{{ {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); }}}} @@ -3987,8 +4199,9 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: goto exit; }}}} }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class double_converter(CConverter): type = 'double' @@ -3996,9 +4209,9 @@ class double_converter(CConverter): format_unit = 'd' c_ignored_default = "0.0" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'd': - return """ + return self.format_code(""" if (PyFloat_CheckExact({argname})) {{{{ {paramname} = PyFloat_AS_DOUBLE({argname}); }}}} @@ -4009,8 +4222,9 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: goto exit; }}}} }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class Py_complex_converter(CConverter): @@ -4019,15 +4233,16 @@ class Py_complex_converter(CConverter): format_unit = 'D' c_ignored_default = "{0.0, 0.0}" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'D': - return """ + return self.format_code(""" {paramname} = PyComplex_AsCComplex({argname}); if (PyErr_Occurred()) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name) - return super().parse_arg(argname, displayname) + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class object_converter(CConverter): @@ -4112,11 +4327,11 @@ def post_parsing(self) -> str: else: return "" - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 's': - return """ + return self.format_code(""" if (!PyUnicode_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); + {bad_argument} goto exit; }}}} Py_ssize_t {length_name}; @@ -4128,10 +4343,12 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname, length_name=self.length_name) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + length_name=self.length_name) if self.format_unit == 'z': - return """ + return self.format_code(""" if ({argname} == Py_None) {{{{ {paramname} = NULL; }}}} @@ -4147,12 +4364,14 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: }}}} }}}} else {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "str or None", {argname}); + {bad_argument} goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname, length_name=self.length_name) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + length_name=self.length_name) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) # # This is the fourth or fifth rewrite of registering all the @@ -4214,51 +4433,57 @@ class PyBytesObject_converter(CConverter): format_unit = 'S' # accept = {bytes} - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'S': - return """ + return self.format_code(""" if (!PyBytes_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "bytes", {argname}); + {bad_argument} goto exit; }}}} {paramname} = ({type}){argname}; - """.format(argname=argname, paramname=self.parser_name, - type=self.type, displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class PyByteArrayObject_converter(CConverter): type = 'PyByteArrayObject *' format_unit = 'Y' # accept = {bytearray} - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'Y': - return """ + return self.format_code(""" if (!PyByteArray_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "bytearray", {argname}); + {bad_argument} goto exit; }}}} {paramname} = ({type}){argname}; - """.format(argname=argname, paramname=self.parser_name, - type=self.type, displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class unicode_converter(CConverter): type = 'PyObject *' default_type = (str, Null, NoneType) format_unit = 'U' - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'U': - return """ + return self.format_code(""" if (!PyUnicode_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); + {bad_argument} goto exit; }}}} {paramname} = {argname}; - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) @add_legacy_c_converter('u') @add_legacy_c_converter('u#', zeroes=True) @@ -4292,25 +4517,26 @@ def cleanup(self) -> str: if self.length: return "" else: - return """\ -PyMem_Free((void *){name}); -""".format(name=self.name) + return f"""PyMem_Free((void *){self.parser_name});\n""" - def parse_arg(self, argname: str, argnum: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if not self.length: if self.accept == {str}: - return """ + return self.format_code(""" if (!PyUnicode_Check({argname})) {{{{ - _PyArg_BadArgument("{{name}}", {argnum}, "str", {argname}); + {bad_argument} goto exit; }}}} {paramname} = PyUnicode_AsWideCharString({argname}, NULL); if ({paramname} == NULL) {{{{ goto exit; }}}} - """.format(argname=argname, paramname=self.name, argnum=argnum) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) elif self.accept == {str, NoneType}: - return """ + return self.format_code(""" if ({argname} == Py_None) {{{{ {paramname} = NULL; }}}} @@ -4321,11 +4547,14 @@ def parse_arg(self, argname: str, argnum: str) -> str | None: }}}} }}}} else {{{{ - _PyArg_BadArgument("{{name}}", {argnum}, "str or None", {argname}); + {bad_argument} goto exit; }}}} - """.format(argname=argname, paramname=self.name, argnum=argnum) - return super().parse_arg(argname, argnum) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) @add_legacy_c_converter('s*', accept={str, buffer}) @add_legacy_c_converter('z*', accept={str, buffer, NoneType}) @@ -4359,20 +4588,22 @@ def cleanup(self) -> str: name = self.name return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) - def parse_arg(self, argname: str, displayname: str) -> str | None: + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'y*': - return """ + return self.format_code(""" if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ goto exit; }}}} if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); + {bad_argument} goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) elif self.format_unit == 's*': - return """ + return self.format_code(""" if (PyUnicode_Check({argname})) {{{{ Py_ssize_t len; const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); @@ -4386,26 +4617,30 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: goto exit; }}}} if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); + {bad_argument} goto exit; }}}} }}}} - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) elif self.format_unit == 'w*': - return """ + return self.format_code(""" if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ - PyErr_Clear(); - _PyArg_BadArgument("{{name}}", {displayname}, "read-write bytes-like object", {argname}); + {bad_argument} goto exit; }}}} if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); + {bad_argument2} goto exit; }}}} - """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) - return super().parse_arg(argname, displayname) + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), + bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) def correct_name_for_self( From 03c4080c71f49df9c219354b7b38b738917fd2ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 3 Sep 2023 18:54:27 +0200 Subject: [PATCH 475/598] gh-108765: Python.h no longer includes (#108831) Remove in C files which don't use it; only sre.c and _decimal.c still use it. Remove _PY_PORT_CTYPE_UTF8_ISSUE code from pyport.h: * Code added by commit b5047fd01948ab108edcc1b3c2c901d915814cfd in 2004 for MacOSX and FreeBSD. * Test removed by commit 52ddaefb6bab1a74ecffe8519c02735794ebfbe1 in 2007, since Python str type now uses locale independent functions like Py_ISALPHA() and Py_TOLOWER() and the Unicode database. Modules/_sre/sre.c replaces _PY_PORT_CTYPE_UTF8_ISSUE with new functions: sre_isalnum(), sre_tolower(), sre_toupper(). Remove unused includes: * _localemodule.c: remove . * getargs.c: remove . * dynload_win.c: remove , it no longer calls _getcwd() since commit fb1f68ed7cc1536482d1debd70a53c5442135fe2 (in 2001). --- Doc/whatsnew/3.13.rst | 7 ++++ Include/Python.h | 1 - Include/pyport.h | 38 ------------------- ...-09-02-22-35-55.gh-issue-108765.4TOdBT.rst | 5 +++ Modules/_localemodule.c | 34 ++++++----------- Modules/_sre/sre.c | 38 ++++++++++++++++--- Modules/_struct.c | 1 - Modules/_tkinter.c | 13 +++---- Modules/_zoneinfo.c | 7 ++-- Modules/pyexpat.c | 1 - Modules/timemodule.c | 7 ++-- Objects/abstract.c | 3 +- Objects/floatobject.c | 3 +- Objects/longobject.c | 6 +-- Objects/typeobject.c | 2 - Python/bltinmodule.c | 1 - Python/ceval.c | 4 +- Python/codecs.c | 1 - Python/dynload_win.c | 9 +---- Python/errors.c | 1 - Python/getargs.c | 4 -- Python/mystrtoul.c | 14 +++++-- 22 files changed, 86 insertions(+), 114 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-02-22-35-55.gh-issue-108765.4TOdBT.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 1c91a1dadb899f..5d8ecbb193f157 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -942,6 +942,13 @@ Porting to Python 3.13 and ``setitimer()`` functions. (Contributed by Victor Stinner in :gh:`108765`.) +* ``Python.h`` no longer includes the ```` standard header file. If + needed, it should now be included explicitly. For example, it provides + ``isalpha()`` and ``tolower()`` functions which are locale dependent. Python + provides locale independent functions, like :c:func:`!Py_ISALPHA` and + :c:func:`!Py_TOLOWER`. + (Contributed by Victor Stinner in :gh:`108765`.) + Deprecated ---------- diff --git a/Include/Python.h b/Include/Python.h index 4cc72bb23ce7a3..8b28200000ab56 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -17,7 +17,6 @@ // Include standard header files #include // assert() -#include // tolower() #include // uintptr_t #include // INT_MAX #include // HUGE_VAL diff --git a/Include/pyport.h b/Include/pyport.h index c4168d10f58151..4d9b9c026b31d4 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -392,44 +392,6 @@ extern "C" { # define Py_NO_INLINE #endif -/* On 4.4BSD-descendants, ctype functions serves the whole range of - * wchar_t character set rather than single byte code points only. - * This characteristic can break some operations of string object - * including str.upper() and str.split() on UTF-8 locales. This - * workaround was provided by Tim Robbins of FreeBSD project. - */ - -#if defined(__APPLE__) -# define _PY_PORT_CTYPE_UTF8_ISSUE -#endif - -#ifdef _PY_PORT_CTYPE_UTF8_ISSUE -#ifndef __cplusplus - /* The workaround below is unsafe in C++ because - * the defines these symbols as real functions, - * with a slightly different signature. - * See issue #10910 - */ -#include -#include -#undef isalnum -#define isalnum(c) iswalnum(btowc(c)) -#undef isalpha -#define isalpha(c) iswalpha(btowc(c)) -#undef islower -#define islower(c) iswlower(btowc(c)) -#undef isspace -#define isspace(c) iswspace(btowc(c)) -#undef isupper -#define isupper(c) iswupper(btowc(c)) -#undef tolower -#define tolower(c) towlower(btowc(c)) -#undef toupper -#define toupper(c) towupper(btowc(c)) -#endif -#endif - - /* Declarations for symbol visibility. PyAPI_FUNC(type): Declares a public Python API function and return type diff --git a/Misc/NEWS.d/next/C API/2023-09-02-22-35-55.gh-issue-108765.4TOdBT.rst b/Misc/NEWS.d/next/C API/2023-09-02-22-35-55.gh-issue-108765.4TOdBT.rst new file mode 100644 index 00000000000000..c13b6d9db053fc --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-02-22-35-55.gh-issue-108765.4TOdBT.rst @@ -0,0 +1,5 @@ +``Python.h`` no longer includes the ```` standard header file. If +needed, it should now be included explicitly. For example, it provides +``isalpha()`` and ``tolower()`` functions which are locale dependent. Python +provides locale independent functions, like :c:func:`!Py_ISALPHA` and +:c:func:`!Py_TOLOWER`. Patch by Victor Stinner. diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 1847a4811e8ee9..fe8e4c5e30035b 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -10,35 +10,25 @@ This software comes with no warranty. Use at your own risk. ******************************************************************/ #include "Python.h" -#include "pycore_fileutils.h" -#include "pycore_pymem.h" // _PyMem_Strdup - -#include -#include -#include -#include +#include "pycore_fileutils.h" // _Py_GetLocaleconvNumeric() +#include "pycore_pymem.h" // _PyMem_Strdup() +#include // setlocale() +#include // strlen() #ifdef HAVE_ERRNO_H -#include +# include // errno #endif - #ifdef HAVE_LANGINFO_H -#include +# include // nl_langinfo() #endif - #ifdef HAVE_LIBINTL_H -#include -#endif - -#ifdef HAVE_WCHAR_H -#include +# include #endif - -#if defined(MS_WINDOWS) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include +#ifdef MS_WINDOWS +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include #endif PyDoc_STRVAR(locale__doc__, "Support for POSIX locales."); diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 3872c3663c7294..07da5da13f70d3 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -43,12 +43,40 @@ static const char copyright[] = #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "sre.h" // SRE_CODE -#include "sre.h" +#include // tolower(), toupper(), isalnum() #define SRE_CODE_BITS (8 * sizeof(SRE_CODE)) -#include +// On macOS, use the wide character ctype API using btowc() +#if defined(__APPLE__) +# define USE_CTYPE_WINT_T +#endif + +static int sre_isalnum(unsigned int ch) { +#ifdef USE_CTYPE_WINT_T + return (unsigned int)iswalnum(btowc((int)ch)); +#else + return (unsigned int)isalnum((int)ch); +#endif +} + +static unsigned int sre_tolower(unsigned int ch) { +#ifdef USE_CTYPE_WINT_T + return (unsigned int)towlower(btowc((int)ch)); +#else + return (unsigned int)tolower((int)ch); +#endif +} + +static unsigned int sre_toupper(unsigned int ch) { +#ifdef USE_CTYPE_WINT_T + return (unsigned int)towupper(btowc((int)ch)); +#else + return (unsigned int)toupper((int)ch); +#endif +} /* Defining this one controls tracing: * 0 -- disabled @@ -114,17 +142,17 @@ static unsigned int sre_lower_ascii(unsigned int ch) /* locale-specific character predicates */ /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids * warnings when c's type supports only numbers < N+1 */ -#define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? isalnum((ch)) : 0) +#define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? sre_isalnum((ch)) : 0) #define SRE_LOC_IS_WORD(ch) (SRE_LOC_IS_ALNUM((ch)) || (ch) == '_') static unsigned int sre_lower_locale(unsigned int ch) { - return ((ch) < 256 ? (unsigned int)tolower((ch)) : ch); + return ((ch) < 256 ? (unsigned int)sre_tolower((ch)) : ch); } static unsigned int sre_upper_locale(unsigned int ch) { - return ((ch) < 256 ? (unsigned int)toupper((ch)) : ch); + return ((ch) < 256 ? (unsigned int)sre_toupper((ch)) : ch); } /* unicode-specific character predicates */ diff --git a/Modules/_struct.c b/Modules/_struct.c index be4c23af384bb9..1f8f9c44dd69aa 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -12,7 +12,6 @@ #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include #include // offsetof() /*[clinic input] diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 663b4117683629..f9a18644945c65 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -26,15 +26,14 @@ Copyright (C) 1994 Steen Lumholt. #endif #include "Python.h" -#include #ifdef MS_WINDOWS # include "pycore_fileutils.h" // _Py_stat() #endif -#include "pycore_long.h" +#include "pycore_long.h" // _PyLong_IsNegative() #ifdef MS_WINDOWS -#include +# include #endif #define CHECK_SIZE(size, elemsize) \ @@ -46,11 +45,11 @@ Copyright (C) 1994 Steen Lumholt. #define TCL_THREADS #ifdef TK_FRAMEWORK -#include -#include +# include +# include #else -#include -#include +# include +# include #endif #include "tkinter.h" diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 3f7b2851c5b794..eb4e522465181f 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -6,11 +6,10 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include -#include -#include +#include "datetime.h" // PyDateTime_TZInfo -#include "datetime.h" +#include // offsetof() +#include #include "clinic/_zoneinfo.c.h" /*[clinic input] diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 52dd06cd3c8181..bd24523eac830b 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -7,7 +7,6 @@ #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() -#include #include // offsetof() #include "expat.h" #include "pyexpat.h" diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 4e55da71b117ca..a2b66520ee885d 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -6,22 +6,21 @@ #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_runtime.h" // _Py_ID() -#include #include // clock() #ifdef HAVE_SYS_TIMES_H -# include +# include // times() #endif #ifdef HAVE_SYS_TYPES_H # include #endif #if defined(HAVE_SYS_RESOURCE_H) -# include +# include // getrusage(RUSAGE_SELF) #endif #ifdef QUICKWIN # include #endif #if defined(HAVE_PTHREAD_H) -# include +# include // pthread_getcpuclockid() #endif #if defined(_AIX) # include diff --git a/Objects/abstract.c b/Objects/abstract.c index c113364a88a26a..6713926b86d218 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -9,9 +9,8 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_unionobject.h" // _PyUnion_Check() -#include -#include // offsetof() +#include // offsetof() /* Shorthands to return certain errors */ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1c5078bdda6871..776c7092edd057 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -16,8 +16,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include -#include +#include // DBL_MAX #include // strtol() /*[clinic input] diff --git a/Objects/longobject.c b/Objects/longobject.c index d20ef412367bb7..e73de742229005 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -10,10 +10,8 @@ #include "pycore_runtime.h" // _PY_NSMALLPOSINTS #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include -#include -#include -#include // abs() +#include // DBL_MANT_DIG +#include // offsetof #include "clinic/longobject.c.h" /*[clinic input] diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 67e059c3f74b77..84c50507691cbe 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -19,8 +19,6 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "opcode.h" // MAKE_CELL - -#include #include // ptrdiff_t /*[clinic input] diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 971067e2d4fcc1..8e234e085f16c9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,7 +1,6 @@ /* Built-in functions */ #include "Python.h" -#include #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_Vector() diff --git a/Python/ceval.c b/Python/ceval.c index a22852ec13ea99..6f90d8e6fd064b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -35,9 +35,7 @@ #include "pydtrace.h" #include "setobject.h" - -#include -#include +#include // bool #ifdef Py_DEBUG /* For debugging the interpreter: */ diff --git a/Python/codecs.c b/Python/codecs.c index 87ae896b8e7718..b79bf555f2f22a 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -14,7 +14,6 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI -#include const char *Py_hexdigits = "0123456789abcdef"; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index acab05e2c6def3..f69995b8f9e3a1 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -5,13 +5,8 @@ #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#ifdef HAVE_DIRECT_H -#include -#endif -#include - -#include "importdl.h" -#include "patchlevel.h" +#include "importdl.h" // dl_funcptr +#include "patchlevel.h" // PY_MAJOR_VERSION #include #ifdef _DEBUG diff --git a/Python/errors.c b/Python/errors.c index fb5b3ff2c7ba51..f670b78c1f14ef 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -10,7 +10,6 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() -#include #ifdef MS_WINDOWS # include # include diff --git a/Python/getargs.c b/Python/getargs.c index fdc144488c9627..cbfe561111176c 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -7,10 +7,6 @@ #include "pycore_pylifecycle.h" // _PyArg_Fini #include "pycore_tuple.h" // _PyTuple_ITEMS() -#include -#include - - #ifdef __cplusplus extern "C" { #endif diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c index e6fe154eed611a..fcd3e27f17f4e9 100644 --- a/Python/mystrtoul.c +++ b/Python/mystrtoul.c @@ -1,16 +1,22 @@ +// strtol() and strtoul(), renamed to avoid conflicts. +// +// API: +// +// - PyOS_strtol(): convert string to C long integer. +// - PyOS_strtoul(): convert string to C unsigned long integer. + #include "Python.h" #include "pycore_long.h" // _PyLong_DigitValue #if defined(__sgi) && !defined(_SGI_MP_SOURCE) -#define _SGI_MP_SOURCE +# define _SGI_MP_SOURCE #endif /* strtol and strtoul, renamed to avoid conflicts */ -#include #ifdef HAVE_ERRNO_H -#include +# include // errno #endif /* Static overflow check values for bases 2 through 36. @@ -75,7 +81,7 @@ static const int digitlimit[] = { 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, /* 20 - 29 */ 13, 12, 12, 12, 12, 12, 12}; /* 30 - 36 */ #else -#error "Need table for SIZEOF_LONG" +# error "Need table for SIZEOF_LONG" #endif /* From 0c369d6cb8c9a73725f5794c84bedf93e46fd27d Mon Sep 17 00:00:00 2001 From: Sangyun_LEE Date: Mon, 4 Sep 2023 06:19:49 +0900 Subject: [PATCH 476/598] Update Lib/test/test_unittest/testmock/testmock.py: fix typo RuntimError to RuntimeError (#108847) --- Lib/test/test_unittest/testmock/testmock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index bb09913d70b7ca..d23eb87696f406 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -2270,7 +2270,7 @@ def test_misspelled_arguments(self): class Foo(): one = 'one' # patch, patch.object and create_autospec need to check for misspelled - # arguments explicitly and throw a RuntimError if found. + # arguments explicitly and throw a RuntimeError if found. with self.assertRaises(RuntimeError): with patch(f'{__name__}.Something.meth', autospect=True): pass with self.assertRaises(RuntimeError): From c2ec174d243da5d2607dbf06c4451d0093ac40ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 3 Sep 2023 23:32:13 +0200 Subject: [PATCH 477/598] gh-108765: Move stat() fiddling from pyport.h to fileutils.h (#108854) --- Include/fileutils.h | 36 ++++++++++++++++++++++++++++++++++++ Include/pyport.h | 37 ------------------------------------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index ba5acc84fcb185..1509198e45f0ca 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -1,5 +1,41 @@ #ifndef Py_FILEUTILS_H #define Py_FILEUTILS_H + +/******************************* + * stat() and fstat() fiddling * + *******************************/ + +#ifdef HAVE_SYS_STAT_H +# include // S_ISREG() +#elif defined(HAVE_STAT_H) +# include // S_ISREG() +#endif + +#ifndef S_IFMT + // VisualAge C/C++ Failed to Define MountType Field in sys/stat.h. +# define S_IFMT 0170000 +#endif +#ifndef S_IFLNK + // Windows doesn't define S_IFLNK, but posixmodule.c maps + // IO_REPARSE_TAG_SYMLINK to S_IFLNK. +# define S_IFLNK 0120000 +#endif +#ifndef S_ISREG +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif +#ifndef S_ISDIR +# define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) +#endif +#ifndef S_ISCHR +# define S_ISCHR(x) (((x) & S_IFMT) == S_IFCHR) +#endif +#ifndef S_ISLNK +# define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) +#endif + + +// Move this down here since some C++ #include's don't like to be included +// inside an extern "C". #ifdef __cplusplus extern "C" { #endif diff --git a/Include/pyport.h b/Include/pyport.h index 4d9b9c026b31d4..4a21a446b3d904 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -184,43 +184,6 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_MEMCPY memcpy #endif -/******************************* - * stat() and fstat() fiddling * - *******************************/ - -#ifdef HAVE_SYS_STAT_H -#include -#elif defined(HAVE_STAT_H) -#include -#endif - -#ifndef S_IFMT -/* VisualAge C/C++ Failed to Define MountType Field in sys/stat.h */ -#define S_IFMT 0170000 -#endif - -#ifndef S_IFLNK -/* Windows doesn't define S_IFLNK but posixmodule.c maps - * IO_REPARSE_TAG_SYMLINK to S_IFLNK */ -# define S_IFLNK 0120000 -#endif - -#ifndef S_ISREG -#define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - -#ifndef S_ISDIR -#define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) -#endif - -#ifndef S_ISCHR -#define S_ISCHR(x) (((x) & S_IFMT) == S_IFCHR) -#endif - -#ifndef S_ISLNK -#define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) -#endif - #ifdef __cplusplus /* Move this down here since some C++ #include's don't like to be included inside an extern "C" */ From 31c2945f143c6b80c837fcf09a5cfb85fea9ea4c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 3 Sep 2023 23:37:15 +0200 Subject: [PATCH 478/598] gh-108834: regrtest reruns failed tests in subprocesses (#108839) When using --rerun option, regrtest now re-runs failed tests in verbose mode in fresh worker processes to have more deterministic behavior. So it can write its final report even if a test killed a worker progress. Add --fail-rerun option to regrtest: exit with non-zero exit code if a test failed pass passed when re-run in verbose mode (in a fresh process). That's now more useful since tests can pass when re-run in a fresh worker progress, whereas they failed when run after other tests when tests are run sequentially. Rename --verbose2 option (-w) to --rerun. Keep --verbose2 as a deprecated alias. Changes: * Fix and enhance statistics in regrtest summary. Add "(filtered)" when --match and/or --ignore options are used. * Add RunTests class. * Add TestResult.get_rerun_match_tests() method * Rewrite code to serialize/deserialize worker arguments as JSON using a new WorkerJob class. * Fix stats when a test is run with --forever --rerun. * If failed test names cannot be parsed, log a warning and don't filter tests. * test_regrtest.test_rerun_success() now uses a marker file, since the test is re-run in a separated process. * Add tests on normalize_test_name() function. * Add test_success() and test_skip() tests to test_regrtest. --- Lib/test/bisect_cmd.py | 7 +- Lib/test/libregrtest/cmdline.py | 11 +- Lib/test/libregrtest/main.py | 548 +++++++++--------- Lib/test/libregrtest/runtest.py | 234 +++++--- Lib/test/libregrtest/runtest_mp.py | 179 ++++-- Lib/test/libregrtest/utils.py | 2 +- Lib/test/support/__init__.py | 1 - Lib/test/support/testresult.py | 3 + Lib/test/test_regrtest.py | 305 +++++++--- ...-09-03-02-01-55.gh-issue-108834.iAwXzj.rst | 6 + ...-09-03-06-17-12.gh-issue-108834.fjV-CJ.rst | 2 + ...-09-03-20-15-49.gh-issue-108834.Osvmhf.rst | 3 + 12 files changed, 821 insertions(+), 480 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-03-02-01-55.gh-issue-108834.iAwXzj.rst create mode 100644 Misc/NEWS.d/next/Tests/2023-09-03-06-17-12.gh-issue-108834.fjV-CJ.rst create mode 100644 Misc/NEWS.d/next/Tests/2023-09-03-20-15-49.gh-issue-108834.Osvmhf.rst diff --git a/Lib/test/bisect_cmd.py b/Lib/test/bisect_cmd.py index 0bdd7a43c03f7b..5cb804bd469dc3 100755 --- a/Lib/test/bisect_cmd.py +++ b/Lib/test/bisect_cmd.py @@ -109,9 +109,10 @@ def parse_args(): def main(): args = parse_args() - if '-w' in args.test_args or '--verbose2' in args.test_args: - print("WARNING: -w/--verbose2 option should not be used to bisect!") - print() + for opt in ('-w', '--rerun', '--verbose2'): + if opt in args.test_args: + print(f"WARNING: {opt} option should not be used to bisect!") + print() if args.input: with open(args.input) as fp: diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index ebe57920d9185c..251fcacb1d14f7 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -156,7 +156,7 @@ def __init__(self, **kwargs) -> None: self.coverdir = 'coverage' self.runleaks = False self.huntrleaks = False - self.verbose2 = False + self.rerun = False self.verbose3 = False self.print_slow = False self.random_seed = None @@ -213,8 +213,10 @@ def _create_parser(): group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', + group.add_argument('-w', '--rerun', action='store_true', help='re-run failed tests in verbose mode') + group.add_argument('--verbose2', action='store_true', dest='rerun', + help='deprecated alias to --rerun') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') group.add_argument('-q', '--quiet', action='store_true', @@ -309,6 +311,9 @@ def _create_parser(): group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') + group.add_argument('--fail-rerun', action='store_true', + help='if a test failed and then passed when re-run, ' + 'mark the tests as failed') group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', help='writes JUnit-style XML results to the specified ' @@ -380,7 +385,7 @@ def _parse_args(args, **kwargs): ns.python = shlex.split(ns.python) if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") - if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): + if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3): parser.error("--pgo/-v don't go together!") if ns.pgo_extended: ns.pgo = True # pgo_extended implies pgo diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 6e6423e156781b..77a4090a826e06 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,11 +11,11 @@ import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, split_test_packages, runtest, get_abs_module, - PROGRESS_MIN_TIME, State) + findtests, split_test_packages, runtest, abs_module_name, + PROGRESS_MIN_TIME, State, MatchTestsDict, RunTests) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests -from test.libregrtest.utils import (removepy, count, format_duration, +from test.libregrtest.utils import (strip_py_suffix, count, format_duration, printlist, get_build_info) from test import support from test.support import TestStats @@ -28,14 +28,6 @@ # Must be smaller than buildbot "1200 seconds without output" limit. EXIT_TIMEOUT = 120.0 -# gh-90681: When rerunning tests, we might need to rerun the whole -# class or module suite if some its life-cycle hooks fail. -# Test level hooks are not affected. -_TEST_LIFECYCLE_HOOKS = frozenset(( - 'setUpClass', 'tearDownClass', - 'setUpModule', 'tearDownModule', -)) - EXITCODE_BAD_TEST = 2 EXITCODE_INTERRUPTED = 130 EXITCODE_ENV_CHANGED = 3 @@ -72,19 +64,22 @@ def __init__(self): # tests self.tests = [] self.selected = [] + self.all_runtests: list[RunTests] = [] # test results - self.good = [] - self.bad = [] - self.skipped = [] - self.resource_denied = [] - self.environment_changed = [] - self.run_no_tests = [] - self.need_rerun = [] - self.rerun = [] - self.first_result = None + self.good: list[str] = [] + self.bad: list[str] = [] + self.rerun_bad: list[str] = [] + self.skipped: list[str] = [] + self.resource_denied: list[str] = [] + self.environment_changed: list[str] = [] + self.run_no_tests: list[str] = [] + self.rerun: list[str] = [] + + self.need_rerun: list[TestResult] = [] + self.first_state: str | None = None self.interrupted = False - self.stats_dict: dict[str, TestStats] = {} + self.total_stats = TestStats() # used by --slow self.test_times = [] @@ -94,7 +89,7 @@ def __init__(self): # used to display the progress bar "[ 3/100]" self.start_time = time.perf_counter() - self.test_count = '' + self.test_count_text = '' self.test_count_width = 1 # used by --single @@ -107,7 +102,6 @@ def __init__(self): # misc self.win_load_tracker = None self.tmp_dir = None - self.worker_test_name = None def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) @@ -115,11 +109,9 @@ def get_executed(self): | set(self.run_no_tests)) def accumulate_result(self, result, rerun=False): + fail_env_changed = self.ns.fail_env_changed test_name = result.test_name - if result.has_meaningful_duration() and not rerun: - self.test_times.append((result.duration, test_name)) - match result.state: case State.PASSED: self.good.append(test_name) @@ -128,25 +120,24 @@ def accumulate_result(self, result, rerun=False): case State.SKIPPED: self.skipped.append(test_name) case State.RESOURCE_DENIED: - self.skipped.append(test_name) self.resource_denied.append(test_name) case State.INTERRUPTED: self.interrupted = True case State.DID_NOT_RUN: self.run_no_tests.append(test_name) case _: - if result.is_failed(self.ns.fail_env_changed): - if not rerun: - self.bad.append(test_name) - self.need_rerun.append(result) + if result.is_failed(fail_env_changed): + self.bad.append(test_name) + self.need_rerun.append(result) else: - raise ValueError(f"invalid test state: {state!r}") + raise ValueError(f"invalid test state: {result.state!r}") + if result.has_meaningful_duration() and not rerun: + self.test_times.append((result.duration, test_name)) if result.stats is not None: - self.stats_dict[result.test_name] = result.stats - - if rerun and not(result.is_failed(False) or result.state == State.INTERRUPTED): - self.bad.remove(test_name) + self.total_stats.accumulate(result.stats) + if rerun: + self.rerun.append(test_name) xml_data = result.xml_data if xml_data: @@ -180,13 +171,15 @@ def log(self, line=''): print(line, flush=True) def display_progress(self, test_index, text): - if self.ns.quiet: + quiet = self.ns.quiet + pgo = self.ns.pgo + if quiet: return # "[ 51/405/1] test_tcl passed" - line = f"{test_index:{self.test_count_width}}{self.test_count}" + line = f"{test_index:{self.test_count_width}}{self.test_count_text}" fails = len(self.bad) + len(self.environment_changed) - if fails and not self.ns.pgo: + if fails and not pgo: line = f"{line}/{fails}" self.log(f"[{line}] {text}") @@ -196,15 +189,7 @@ def parse_args(self, kwargs): if ns.xmlpath: support.junit_xml_list = self.testsuite_xml = [] - worker_args = ns.worker_args - if worker_args is not None: - from test.libregrtest.runtest_mp import parse_worker_args - ns, test_name = parse_worker_args(ns.worker_args) - ns.worker_args = worker_args - self.worker_test_name = test_name - - # Strip .py extensions. - removepy(ns.args) + strip_py_suffix(ns.args) if ns.huntrleaks: warmup, repetitions, _ = ns.huntrleaks @@ -221,9 +206,18 @@ def parse_args(self, kwargs): self.ns = ns def find_tests(self, tests): + ns = self.ns + single = ns.single + fromfile = ns.fromfile + pgo = ns.pgo + exclude = ns.exclude + test_dir = ns.testdir + starting_test = ns.start + randomize = ns.randomize + self.tests = tests - if self.ns.single: + if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: with open(self.next_single_filename, 'r') as fp: @@ -232,12 +226,12 @@ def find_tests(self, tests): except OSError: pass - if self.ns.fromfile: + if fromfile: self.tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') - with open(os.path.join(os_helper.SAVEDCWD, self.ns.fromfile)) as fp: + with open(os.path.join(os_helper.SAVEDCWD, fromfile)) as fp: for line in fp: line = line.split('#', 1)[0] line = line.strip() @@ -245,22 +239,22 @@ def find_tests(self, tests): if match is not None: self.tests.append(match.group()) - removepy(self.tests) + strip_py_suffix(self.tests) - if self.ns.pgo: + if pgo: # add default PGO tests if no tests are specified - setup_pgo_tests(self.ns) + setup_pgo_tests(ns) - exclude = set() - if self.ns.exclude: - for arg in self.ns.args: - exclude.add(arg) - self.ns.args = [] + exclude_tests = set() + if exclude: + for arg in ns.args: + exclude_tests.add(arg) + ns.args = [] - alltests = findtests(testdir=self.ns.testdir, exclude=exclude) + alltests = findtests(testdir=test_dir, exclude=exclude_tests) - if not self.ns.fromfile: - self.selected = self.tests or self.ns.args + if not fromfile: + self.selected = self.tests or ns.args if self.selected: self.selected = split_test_packages(self.selected) else: @@ -268,7 +262,7 @@ def find_tests(self, tests): else: self.selected = self.tests - if self.ns.single: + if single: self.selected = self.selected[:1] try: pos = alltests.index(self.selected[0]) @@ -277,17 +271,17 @@ def find_tests(self, tests): pass # Remove all the selected tests that precede start if it's set. - if self.ns.start: + if starting_test: try: - del self.selected[:self.selected.index(self.ns.start)] + del self.selected[:self.selected.index(starting_test)] except ValueError: - print("Couldn't find starting test (%s), using all tests" - % self.ns.start, file=sys.stderr) + print(f"Cannot find starting test: {starting_test}") + sys.exit(1) - if self.ns.randomize: - if self.ns.random_seed is None: - self.ns.random_seed = random.randrange(10000000) - random.seed(self.ns.random_seed) + if randomize: + if ns.random_seed is None: + ns.random_seed = random.randrange(10000000) + random.seed(ns.random_seed) random.shuffle(self.selected) def list_tests(self): @@ -305,25 +299,63 @@ def _list_cases(self, suite): print(test.id()) def list_cases(self): + ns = self.ns + test_dir = ns.testdir support.verbose = False - support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests) + support.set_match_tests(ns.match_tests, ns.ignore_tests) + skipped = [] for test_name in self.selected: - abstest = get_abs_module(self.ns, test_name) + module_name = abs_module_name(test_name, test_dir) try: - suite = unittest.defaultTestLoader.loadTestsFromName(abstest) + suite = unittest.defaultTestLoader.loadTestsFromName(module_name) self._list_cases(suite) except unittest.SkipTest: - self.skipped.append(test_name) + skipped.append(test_name) - if self.skipped: - print(file=sys.stderr) - print(count(len(self.skipped), "test"), "skipped:", file=sys.stderr) - printlist(self.skipped, file=sys.stderr) + if skipped: + sys.stdout.flush() + stderr = sys.stderr + print(file=stderr) + print(count(len(skipped), "test"), "skipped:", file=stderr) + printlist(skipped, file=stderr) - def rerun_failed_tests(self): - self.log() + def get_rerun_match(self, rerun_list) -> MatchTestsDict: + rerun_match_tests = {} + for result in rerun_list: + match_tests = result.get_rerun_match_tests() + # ignore empty match list + if match_tests: + rerun_match_tests[result.test_name] = match_tests + return rerun_match_tests + + def _rerun_failed_tests(self, need_rerun): + # Configure the runner to re-run tests + ns = self.ns + ns.verbose = True + ns.failfast = False + ns.verbose3 = False + ns.forever = False + if ns.use_mp is None: + ns.use_mp = 1 + + # Get tests to re-run + tests = [result.test_name for result in need_rerun] + match_tests = self.get_rerun_match(need_rerun) + self.set_tests(tests) + + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.need_rerun.clear() + + # Re-run failed tests + self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") + runtests = RunTests(tests, match_tests=match_tests, rerun=True) + self.all_runtests.append(runtests) + self._run_tests_mp(runtests) + def rerun_failed_tests(self, need_rerun): if self.ns.python: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -332,45 +364,10 @@ def rerun_failed_tests(self): ) return - self.ns.verbose = True - self.ns.failfast = False - self.ns.verbose3 = False - - self.first_result = self.get_tests_result() - - self.log("Re-running failed tests in verbose mode") - rerun_list = list(self.need_rerun) - self.need_rerun.clear() - for result in rerun_list: - test_name = result.test_name - self.rerun.append(test_name) + self.first_state = self.get_tests_state() - errors = result.errors or [] - failures = result.failures or [] - error_names = [ - self.normalize_test_name(test_full_name, is_error=True) - for (test_full_name, *_) in errors] - failure_names = [ - self.normalize_test_name(test_full_name) - for (test_full_name, *_) in failures] - self.ns.verbose = True - orig_match_tests = self.ns.match_tests - if errors or failures: - if self.ns.match_tests is None: - self.ns.match_tests = [] - self.ns.match_tests.extend(error_names) - self.ns.match_tests.extend(failure_names) - matching = "matching: " + ", ".join(self.ns.match_tests) - self.log(f"Re-running {test_name} in verbose mode ({matching})") - else: - self.log(f"Re-running {test_name} in verbose mode") - result = runtest(self.ns, test_name) - self.ns.match_tests = orig_match_tests - - self.accumulate_result(result, rerun=True) - - if result.state == State.INTERRUPTED: - break + print() + self._rerun_failed_tests(need_rerun) if self.bad: print(count(len(self.bad), 'test'), "failed again:") @@ -378,28 +375,17 @@ def rerun_failed_tests(self): self.display_result() - def normalize_test_name(self, test_full_name, *, is_error=False): - short_name = test_full_name.split(" ")[0] - if is_error and short_name in _TEST_LIFECYCLE_HOOKS: - # This means that we have a failure in a life-cycle hook, - # we need to rerun the whole module or class suite. - # Basically the error looks like this: - # ERROR: setUpClass (test.test_reg_ex.RegTest) - # or - # ERROR: setUpModule (test.test_reg_ex) - # So, we need to parse the class / module name. - lpar = test_full_name.index('(') - rpar = test_full_name.index(')') - return test_full_name[lpar + 1: rpar].split('.')[-1] - return short_name - def display_result(self): + pgo = self.ns.pgo + quiet = self.ns.quiet + print_slow = self.ns.print_slow + # If running the test suite for PGO then no one cares about results. - if self.ns.pgo: + if pgo: return print() - print("== Tests result: %s ==" % self.get_tests_result()) + print("== Tests result: %s ==" % self.get_tests_state()) if self.interrupted: print("Test suite interrupted by signal SIGINT.") @@ -410,7 +396,7 @@ def display_result(self): print(count(len(omitted), "test"), "omitted:") printlist(omitted) - if self.good and not self.ns.quiet: + if self.good and not quiet: print() if (not self.bad and not self.skipped @@ -419,7 +405,7 @@ def display_result(self): print("All", end=' ') print(count(len(self.good), "test"), "OK.") - if self.ns.print_slow: + if print_slow: self.test_times.sort(reverse=True) print() print("10 slowest tests:") @@ -437,11 +423,16 @@ def display_result(self): count(len(self.environment_changed), "test"))) printlist(self.environment_changed) - if self.skipped and not self.ns.quiet: + if self.skipped and not quiet: print() print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) + if self.resource_denied and not quiet: + print() + print(count(len(self.resource_denied), "test"), "skipped (resource denied):") + printlist(self.resource_denied) + if self.rerun: print() print("%s:" % count(len(self.rerun), "re-run test")) @@ -452,40 +443,58 @@ def display_result(self): print(count(len(self.run_no_tests), "test"), "run no tests:") printlist(self.run_no_tests) - def run_tests_sequential(self): - if self.ns.trace: + def run_test(self, test_index, test_name, previous_test, save_modules): + text = test_name + if previous_test: + text = '%s -- %s' % (text, previous_test) + self.display_progress(test_index, text) + + if self.tracer: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + cmd = ('result = runtest(self.ns, test_name); ' + 'self.accumulate_result(result)') + ns = dict(locals()) + self.tracer.runctx(cmd, globals=globals(), locals=ns) + result = ns['result'] + else: + result = runtest(self.ns, test_name) + self.accumulate_result(result) + + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) + + return result + + def run_tests_sequentially(self, runtests): + ns = self.ns + coverage = ns.trace + fail_fast = ns.failfast + fail_env_changed = ns.fail_env_changed + timeout = ns.timeout + + if coverage: import trace self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() msg = "Run tests sequentially" - if self.ns.timeout: - msg += " (timeout: %s)" % format_duration(self.ns.timeout) + if timeout: + msg += " (timeout: %s)" % format_duration(timeout) self.log(msg) previous_test = None - for test_index, test_name in enumerate(self.tests, 1): + tests_iter = runtests.iter_tests() + for test_index, test_name in enumerate(tests_iter, 1): start_time = time.perf_counter() - text = test_name - if previous_test: - text = '%s -- %s' % (text, previous_test) - self.display_progress(test_index, text) - - if self.tracer: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name); ' - 'self.accumulate_result(result)') - ns = dict(locals()) - self.tracer.runctx(cmd, globals=globals(), locals=ns) - result = ns['result'] - else: - result = runtest(self.ns, test_name) - self.accumulate_result(result) + result = self.run_test(test_index, test_name, + previous_test, save_modules) - if result.state == State.INTERRUPTED: + if result.must_stop(fail_fast, fail_env_changed): break previous_test = str(result) @@ -496,26 +505,9 @@ def run_tests_sequential(self): # be quiet: say nothing if the test passed shortly previous_test = None - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) - - if self.ns.failfast and result.is_failed(self.ns.fail_env_changed): - break - if previous_test: print(previous_test) - def _test_forever(self, tests): - while True: - for test_name in tests: - yield test_name - if self.bad: - return - if self.ns.fail_env_changed and self.environment_changed: - return - def display_header(self): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) @@ -560,11 +552,13 @@ def no_tests_run(self): return not any((self.good, self.bad, self.skipped, self.interrupted, self.environment_changed)) - def get_tests_result(self): + def get_tests_state(self): + fail_env_changed = self.ns.fail_env_changed + result = [] if self.bad: result.append("FAILURE") - elif self.ns.fail_env_changed and self.environment_changed: + elif fail_env_changed and self.environment_changed: result.append("ENV CHANGED") elif self.no_tests_run(): result.append("NO TESTS RAN") @@ -576,10 +570,40 @@ def get_tests_result(self): result.append("SUCCESS") result = ', '.join(result) - if self.first_result: - result = '%s then %s' % (self.first_result, result) + if self.first_state: + result = '%s then %s' % (self.first_state, result) return result + def _run_tests_mp(self, runtests: RunTests) -> None: + from test.libregrtest.runtest_mp import run_tests_multiprocess + # If we're on windows and this is the parent runner (not a worker), + # track the load average. + if sys.platform == 'win32': + from test.libregrtest.win_utils import WindowsLoadTracker + + try: + self.win_load_tracker = WindowsLoadTracker() + except PermissionError as error: + # Standard accounts may not have access to the performance + # counters. + print(f'Failed to create WindowsLoadTracker: {error}') + + try: + run_tests_multiprocess(self, runtests) + finally: + if self.win_load_tracker is not None: + self.win_load_tracker.close() + self.win_load_tracker = None + + def set_tests(self, tests): + self.tests = tests + if self.ns.forever: + self.test_count_text = '' + self.test_count_width = 3 + else: + self.test_count_text = '/{}'.format(len(self.tests)) + self.test_count_width = len(self.test_count_text) - 1 + def run_tests(self): # For a partial run, we do not need to clutter the output. if (self.ns.header @@ -597,37 +621,14 @@ def run_tests(self): if self.ns.randomize: print("Using random seed", self.ns.random_seed) - if self.ns.forever: - self.tests = self._test_forever(list(self.selected)) - self.test_count = '' - self.test_count_width = 3 - else: - self.tests = iter(self.selected) - self.test_count = '/{}'.format(len(self.selected)) - self.test_count_width = len(self.test_count) - 1 - + tests = self.selected + self.set_tests(tests) + runtests = RunTests(tests, forever=self.ns.forever) + self.all_runtests.append(runtests) if self.ns.use_mp: - from test.libregrtest.runtest_mp import run_tests_multiprocess - # If we're on windows and this is the parent runner (not a worker), - # track the load average. - if sys.platform == 'win32' and self.worker_test_name is None: - from test.libregrtest.win_utils import WindowsLoadTracker - - try: - self.win_load_tracker = WindowsLoadTracker() - except PermissionError as error: - # Standard accounts may not have access to the performance - # counters. - print(f'Failed to create WindowsLoadTracker: {error}') - - try: - run_tests_multiprocess(self) - finally: - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None + self._run_tests_mp(runtests) else: - self.run_tests_sequential() + self.run_tests_sequentially(runtests) def finalize(self): if self.next_single_filename: @@ -642,23 +643,29 @@ def finalize(self): r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) - print() - self.display_summary() - if self.ns.runleaks: os.system("leaks %d" % os.getpid()) + self.save_xml_result() + def display_summary(self): duration = time.perf_counter() - self.start_time + first_runtests = self.all_runtests[0] + # the second runtests (re-run failed tests) disables forever, + # use the first runtests + forever = first_runtests.forever + filtered = bool(self.ns.match_tests) or bool(self.ns.ignore_tests) # Total duration + print() print("Total duration: %s" % format_duration(duration)) # Total tests - total = TestStats() - for stats in self.stats_dict.values(): - total.accumulate(stats) - stats = [f'run={total.tests_run:,}'] + total = self.total_stats + text = f'run={total.tests_run:,}' + if filtered: + text = f"{text} (filtered)" + stats = [text] if total.failures: stats.append(f'failures={total.failures:,}') if total.skipped: @@ -666,23 +673,31 @@ def display_summary(self): print(f"Total tests: {' '.join(stats)}") # Total test files - report = [f'success={len(self.good)}'] - if self.bad: - report.append(f'failed={len(self.bad)}') - if self.environment_changed: - report.append(f'env_changed={len(self.environment_changed)}') - if self.skipped: - report.append(f'skipped={len(self.skipped)}') - if self.resource_denied: - report.append(f'resource_denied={len(self.resource_denied)}') - if self.rerun: - report.append(f'rerun={len(self.rerun)}') - if self.run_no_tests: - report.append(f'run_no_tests={len(self.run_no_tests)}') + all_tests = [self.good, self.bad, self.rerun, + self.skipped, + self.environment_changed, self.run_no_tests] + run = sum(map(len, all_tests)) + text = f'run={run}' + if not forever: + ntest = len(first_runtests.tests) + text = f"{text}/{ntest}" + if filtered: + text = f"{text} (filtered)" + report = [text] + for name, tests in ( + ('failed', self.bad), + ('env_changed', self.environment_changed), + ('skipped', self.skipped), + ('resource_denied', self.resource_denied), + ('rerun', self.rerun), + ('run_no_tests', self.run_no_tests), + ): + if tests: + report.append(f'{name}={len(tests)}') print(f"Total test files: {' '.join(report)}") # Result - result = self.get_tests_result() + result = self.get_tests_state() print(f"Result: {result}") def save_xml_result(self): @@ -742,6 +757,9 @@ def set_temp_dir(self): self.tmp_dir = os.path.abspath(self.tmp_dir) + def is_worker(self): + return (self.ns.worker_args is not None) + def create_temp_dir(self): os.makedirs(self.tmp_dir, exist_ok=True) @@ -754,7 +772,8 @@ def create_temp_dir(self): nounce = random.randint(0, 1_000_000) else: nounce = os.getpid() - if self.worker_test_name is not None: + + if self.is_worker(): test_cwd = 'test_python_worker_{}'.format(nounce) else: test_cwd = 'test_python_{}'.format(nounce) @@ -817,48 +836,53 @@ def getloadavg(self): return None + def get_exitcode(self): + exitcode = 0 + if self.bad: + exitcode = EXITCODE_BAD_TEST + elif self.interrupted: + exitcode = EXITCODE_INTERRUPTED + elif self.ns.fail_env_changed and self.environment_changed: + exitcode = EXITCODE_ENV_CHANGED + elif self.no_tests_run(): + exitcode = EXITCODE_NO_TESTS_RAN + elif self.rerun and self.ns.fail_rerun: + exitcode = EXITCODE_BAD_TEST + return exitcode + + def action_run_tests(self): + self.run_tests() + self.display_result() + + need_rerun = self.need_rerun + if self.ns.rerun and need_rerun: + self.rerun_failed_tests(need_rerun) + + self.display_summary() + self.finalize() + def _main(self, tests, kwargs): - if self.worker_test_name is not None: + if self.is_worker(): from test.libregrtest.runtest_mp import run_tests_worker - run_tests_worker(self.ns, self.worker_test_name) + run_tests_worker(self.ns.worker_args) + return if self.ns.wait: input("Press any key to continue...") - support.PGO = self.ns.pgo - support.PGO_EXTENDED = self.ns.pgo_extended - setup_tests(self.ns) - self.find_tests(tests) + exitcode = 0 if self.ns.list_tests: self.list_tests() - sys.exit(0) - - if self.ns.list_cases: + elif self.ns.list_cases: self.list_cases() - sys.exit(0) - - self.run_tests() - self.display_result() - - if self.ns.verbose2 and self.bad: - self.rerun_failed_tests() - - self.finalize() - - self.save_xml_result() + else: + self.action_run_tests() + exitcode = self.get_exitcode() - if self.bad: - sys.exit(EXITCODE_BAD_TEST) - if self.interrupted: - sys.exit(EXITCODE_INTERRUPTED) - if self.ns.fail_env_changed and self.environment_changed: - sys.exit(EXITCODE_ENV_CHANGED) - if self.no_tests_run(): - sys.exit(EXITCODE_NO_TESTS_RAN) - sys.exit(0) + sys.exit(exitcode) def main(tests=None, **kwargs): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 6fa60697371b72..6e3fab1a88318f 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -1,7 +1,6 @@ import dataclasses import doctest import faulthandler -import functools import gc import importlib import io @@ -20,6 +19,10 @@ from test.libregrtest.utils import clear_caches, format_duration, print_warning +MatchTests = list[str] +MatchTestsDict = dict[str, MatchTests] + + # Avoid enum.Enum to reduce the number of imports when tests are run class State: PASSED = "PASSED" @@ -56,6 +59,41 @@ def has_meaningful_duration(state): State.MULTIPROCESSING_ERROR, State.DID_NOT_RUN} + @staticmethod + def must_stop(state): + return state in { + State.INTERRUPTED, + State.MULTIPROCESSING_ERROR} + + +# gh-90681: When rerunning tests, we might need to rerun the whole +# class or module suite if some its life-cycle hooks fail. +# Test level hooks are not affected. +_TEST_LIFECYCLE_HOOKS = frozenset(( + 'setUpClass', 'tearDownClass', + 'setUpModule', 'tearDownModule', +)) + +def normalize_test_name(test_full_name, *, is_error=False): + short_name = test_full_name.split(" ")[0] + if is_error and short_name in _TEST_LIFECYCLE_HOOKS: + if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): + # if setUpModule() or tearDownModule() failed, don't filter + # tests with the test file name, don't use use filters. + return None + + # This means that we have a failure in a life-cycle hook, + # we need to rerun the whole module or class suite. + # Basically the error looks like this: + # ERROR: setUpClass (test.test_reg_ex.RegTest) + # or + # ERROR: setUpModule (test.test_reg_ex) + # So, we need to parse the class / module name. + lpar = test_full_name.index('(') + rpar = test_full_name.index(')') + return test_full_name[lpar + 1: rpar].split('.')[-1] + return short_name + @dataclasses.dataclass(slots=True) class TestResult: @@ -129,6 +167,58 @@ def set_env_changed(self): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED + def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: + if State.must_stop(self.state): + return True + if fail_fast and self.is_failed(fail_env_changed): + return True + return False + + def get_rerun_match_tests(self): + match_tests = [] + + errors = self.errors or [] + failures = self.failures or [] + for error_list, is_error in ( + (errors, True), + (failures, False), + ): + for full_name, *_ in error_list: + match_name = normalize_test_name(full_name, is_error=is_error) + if match_name is None: + # 'setUpModule (test.test_sys)': don't filter tests + return None + if not match_name: + error_type = "ERROR" if is_error else "FAIL" + print_warning(f"rerun failed to parse {error_type} test name: " + f"{full_name!r}: don't filter tests") + return None + match_tests.append(match_name) + + return match_tests + + +@dataclasses.dataclass(slots=True, frozen=True) +class RunTests: + tests: list[str] + match_tests: MatchTestsDict | None = None + rerun: bool = False + forever: bool = False + + def get_match_tests(self, test_name) -> MatchTests | None: + if self.match_tests is not None: + return self.match_tests.get(test_name, None) + else: + return None + + def iter_tests(self): + tests = tuple(self.tests) + if self.forever: + while True: + yield from tests + else: + yield from tests + # Minimum duration of a test to display its duration or to mention that # the test is running in background @@ -147,9 +237,6 @@ def set_env_changed(self): "test_multiprocessing_spawn", } -# Storage of uncollectable objects -FOUND_GARBAGE = [] - def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir @@ -189,31 +276,41 @@ def split_test_packages(tests, *, testdir=None, exclude=(), return splitted -def get_abs_module(ns: Namespace, test_name: str) -> str: - if test_name.startswith('test.') or ns.testdir: +def abs_module_name(test_name: str, test_dir: str | None) -> str: + if test_name.startswith('test.') or test_dir: return test_name else: # Import it from the test package return 'test.' + test_name -def _runtest_capture_output_timeout_junit(result: TestResult, ns: Namespace) -> None: +def setup_support(ns: Namespace): + support.PGO = ns.pgo + support.PGO_EXTENDED = ns.pgo_extended + support.set_match_tests(ns.match_tests, ns.ignore_tests) + support.failfast = ns.failfast + support.verbose = ns.verbose + if ns.xmlpath: + support.junit_xml_list = [] + else: + support.junit_xml_list = None + + +def _runtest(result: TestResult, ns: Namespace) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. - + verbose = ns.verbose output_on_failure = ns.verbose3 + timeout = ns.timeout use_timeout = ( - ns.timeout is not None and threading_helper.can_start_thread + timeout is not None and threading_helper.can_start_thread ) if use_timeout: - faulthandler.dump_traceback_later(ns.timeout, exit=True) + faulthandler.dump_traceback_later(timeout, exit=True) try: - support.set_match_tests(ns.match_tests, ns.ignore_tests) - support.junit_xml_list = xml_list = [] if ns.xmlpath else None - if ns.failfast: - support.failfast = True + setup_support(ns) if output_on_failure: support.verbose = True @@ -247,11 +344,10 @@ def _runtest_capture_output_timeout_junit(result: TestResult, ns: Namespace) -> sys.stderr.flush() else: # Tell tests to be moderately quiet - support.verbose = ns.verbose - - _runtest_env_changed_exc(result, ns, - display_failure=not ns.verbose) + support.verbose = verbose + _runtest_env_changed_exc(result, ns, display_failure=not verbose) + xml_list = support.junit_xml_list if xml_list: import xml.etree.ElementTree as ET result.xml_data = [ET.tostring(x).decode('us-ascii') @@ -276,7 +372,7 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: start_time = time.perf_counter() result = TestResult(test_name) try: - _runtest_capture_output_timeout_junit(result, ns) + _runtest(result, ns) except: if not ns.pgo: msg = traceback.format_exc() @@ -287,9 +383,9 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: return result -def _test_module(the_module): +def run_unittest(test_mod): loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) + tests = loader.loadTestsFromModule(test_mod) for error in loader.errors: print(error, file=sys.stderr) if loader.errors: @@ -304,7 +400,6 @@ def save_env(ns: Namespace, test_name: str): def regrtest_runner(result, test_func, ns) -> None: # Run test_func(), collect statistics, and detect reference and memory # leaks. - if ns.huntrleaks: from test.libregrtest.refleak import dash_R refleak, test_result = dash_R(ns, result.test_name, test_func) @@ -332,24 +427,27 @@ def regrtest_runner(result, test_func, ns) -> None: result.stats = stats +# Storage of uncollectable objects +FOUND_GARBAGE = [] + + def _load_run_test(result: TestResult, ns: Namespace) -> None: # Load the test function, run the test function. + module_name = abs_module_name(result.test_name, ns.testdir) - abstest = get_abs_module(ns, result.test_name) - - # remove the module from sys.module to reload it if it was already imported - try: - del sys.modules[abstest] - except KeyError: - pass + # Remove the module from sys.module to reload it if it was already imported + sys.modules.pop(module_name, None) - the_module = importlib.import_module(abstest) + test_mod = importlib.import_module(module_name) # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_func = getattr(the_module, "test_main", None) - if test_func is None: - test_func = functools.partial(_test_module, the_module) + # tests. If not, use normal unittest test runner. + test_main = getattr(test_mod, "test_main", None) + if test_main is not None: + test_func = test_main + else: + def test_func(): + return run_unittest(test_mod) try: with save_env(ns, result.test_name): @@ -361,12 +459,12 @@ def _load_run_test(result: TestResult, ns: Namespace) -> None: # failures. support.gc_collect() - cleanup_test_droppings(result.test_name, ns.verbose) + remove_testfn(result.test_name, ns.verbose) if gc.garbage: support.environment_altered = True print_warning(f"{result.test_name} created {len(gc.garbage)} " - f"uncollectable object(s).") + f"uncollectable object(s)") # move the uncollectable objects somewhere, # so we don't see them again @@ -444,35 +542,37 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, result.state = State.PASSED -def cleanup_test_droppings(test_name: str, verbose: int) -> None: - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (os_helper.TESTFN,): - if not os.path.exists(name): - continue +def remove_testfn(test_name: str, verbose: int) -> None: + # Try to clean up os_helper.TESTFN if left behind. + # + # While tests shouldn't leave any files or directories behind, when a test + # fails that can be tedious for it to arrange. The consequences can be + # especially nasty on Windows, since if a test leaves a file open, it + # cannot be deleted by name (while there's nothing we can do about that + # here either, we can display the name of the offending test, which is a + # real help). + name = os_helper.TESTFN + if not os.path.exists(name): + return - if os.path.isdir(name): - import shutil - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise RuntimeError(f"os.path says {name!r} exists but is neither " - f"directory nor file") - - if verbose: - print_warning(f"{test_name} left behind {kind} {name!r}") - support.environment_altered = True - - try: - import stat - # fix possible permissions problems that might prevent cleanup - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as exc: - print_warning(f"{test_name} left behind {kind} {name!r} " - f"and it couldn't be removed: {exc}") + if os.path.isdir(name): + import shutil + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise RuntimeError(f"os.path says {name!r} exists but is neither " + f"directory nor file") + + if verbose: + print_warning(f"{test_name} left behind {kind} {name!r}") + support.environment_altered = True + + try: + import stat + # fix possible permissions problems that might prevent cleanup + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as exc: + print_warning(f"{test_name} left behind {kind} {name!r} " + f"and it couldn't be removed: {exc}") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index fb1f80b0c054e3..60089554cab5dd 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -19,8 +19,8 @@ from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - runtest, TestResult, State, - PROGRESS_MIN_TIME) + runtest, TestResult, State, PROGRESS_MIN_TIME, + MatchTests, RunTests) from test.libregrtest.setup import setup_tests from test.libregrtest.utils import format_duration, print_warning @@ -44,26 +44,54 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) -def must_stop(result: TestResult, ns: Namespace) -> bool: - if result.state == State.INTERRUPTED: - return True - if ns.failfast and result.is_failed(ns.fail_env_changed): - return True - return False +@dataclasses.dataclass(slots=True) +class WorkerJob: + test_name: str + namespace: Namespace + rerun: bool = False + match_tests: MatchTests | None = None -def parse_worker_args(worker_args) -> tuple[Namespace, str]: - ns_dict, test_name = json.loads(worker_args) - ns = Namespace(**ns_dict) - return (ns, test_name) +class _EncodeWorkerJob(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + match o: + case WorkerJob(): + result = dataclasses.asdict(o) + result["__worker_job__"] = True + return result + case Namespace(): + result = vars(o) + result["__namespace__"] = True + return result + case _: + return super().default(o) + + +def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: + if "__worker_job__" in d: + d.pop('__worker_job__') + return WorkerJob(**d) + if "__namespace__" in d: + d.pop('__namespace__') + return Namespace(**d) + else: + return d + + +def _parse_worker_args(worker_json: str) -> tuple[Namespace, str]: + return json.loads(worker_json, + object_hook=_decode_worker_job) -def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str, stdout_fh: TextIO) -> subprocess.Popen: - ns_dict = vars(ns) - worker_args = (ns_dict, testname) - worker_args = json.dumps(worker_args) - if ns.python is not None: - executable = ns.python +def run_test_in_subprocess(worker_job: WorkerJob, + output_file: TextIO, + tmp_dir: str | None = None) -> subprocess.Popen: + ns = worker_job.namespace + python = ns.python + worker_args = json.dumps(worker_job, cls=_EncodeWorkerJob) + + if python is not None: + executable = python else: executable = [sys.executable] cmd = [*executable, *support.args_from_interpreter_flags(), @@ -82,9 +110,9 @@ def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str, stdout_fh # sysconfig.is_python_build() is true. See issue 15300. kw = dict( env=env, - stdout=stdout_fh, + stdout=output_file, # bpo-45410: Write stderr into stdout to keep messages order - stderr=stdout_fh, + stderr=output_file, text=True, close_fds=(os.name != 'nt'), cwd=os_helper.SAVEDCWD, @@ -94,11 +122,27 @@ def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str, stdout_fh return subprocess.Popen(cmd, **kw) -def run_tests_worker(ns: Namespace, test_name: str) -> NoReturn: +def run_tests_worker(worker_json: str) -> NoReturn: + worker_job = _parse_worker_args(worker_json) + ns = worker_job.namespace + test_name = worker_job.test_name + rerun = worker_job.rerun + match_tests = worker_job.match_tests + setup_tests(ns) - result = runtest(ns, test_name) + if rerun: + if match_tests: + matching = "matching: " + ", ".join(match_tests) + print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) + else: + print(f"Re-running {test_name} in verbose mode", flush=True) + ns.verbose = True + if match_tests is not None: + ns.match_tests = match_tests + + result = runtest(ns, test_name) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON @@ -148,11 +192,13 @@ class TestWorkerProcess(threading.Thread): def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: super().__init__() self.worker_id = worker_id + self.runtests = runner.runtests self.pending = runner.pending self.output = runner.output self.ns = runner.ns self.timeout = runner.worker_timeout self.regrtest = runner.regrtest + self.rerun = runner.rerun self.current_test_name = None self.start_time = None self._popen = None @@ -216,10 +262,11 @@ def mp_result_error( ) -> MultiprocessResult: return MultiprocessResult(test_result, stdout, err_msg) - def _run_process(self, test_name: str, tmp_dir: str, stdout_fh: TextIO) -> int: - self.current_test_name = test_name + def _run_process(self, worker_job, output_file: TextIO, + tmp_dir: str | None = None) -> int: + self.current_test_name = worker_job.test_name try: - popen = run_test_in_subprocess(test_name, self.ns, tmp_dir, stdout_fh) + popen = run_test_in_subprocess(worker_job, output_file, tmp_dir) self._killed = False self._popen = popen @@ -277,9 +324,15 @@ def _runtest(self, test_name: str) -> MultiprocessResult: else: encoding = sys.stdout.encoding + match_tests = self.runtests.get_match_tests(test_name) + # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. - with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_fh: + with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file: + worker_job = WorkerJob(test_name, + namespace=self.ns, + rerun=self.rerun, + match_tests=match_tests) # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during # Python finalization: too late for libregrtest. @@ -290,17 +343,17 @@ def _runtest(self, test_name: str) -> MultiprocessResult: tmp_dir = tempfile.mkdtemp(prefix="test_python_") tmp_dir = os.path.abspath(tmp_dir) try: - retcode = self._run_process(test_name, tmp_dir, stdout_fh) + retcode = self._run_process(worker_job, stdout_file, tmp_dir) finally: tmp_files = os.listdir(tmp_dir) os_helper.rmtree(tmp_dir) else: - retcode = self._run_process(test_name, None, stdout_fh) + retcode = self._run_process(worker_job, stdout_file) tmp_files = () - stdout_fh.seek(0) + stdout_file.seek(0) try: - stdout = stdout_fh.read().strip() + stdout = stdout_file.read().strip() except Exception as exc: # gh-101634: Catch UnicodeDecodeError if stdout cannot be # decoded from encoding @@ -342,6 +395,8 @@ def _runtest(self, test_name: str) -> MultiprocessResult: return MultiprocessResult(result, stdout) def run(self) -> None: + fail_fast = self.ns.failfast + fail_env_changed = self.ns.fail_env_changed while not self._stopped: try: try: @@ -354,7 +409,7 @@ def run(self) -> None: mp_result.result.duration = time.monotonic() - self.start_time self.output.put((False, mp_result)) - if must_stop(mp_result.result, self.ns): + if mp_result.result.must_stop(fail_fast, fail_env_changed): break except ExitThread: break @@ -410,29 +465,36 @@ def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: class MultiprocessTestRunner: - def __init__(self, regrtest: Regrtest) -> None: + def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: + ns = regrtest.ns + timeout = ns.timeout + self.regrtest = regrtest + self.runtests = runtests + self.rerun = runtests.rerun self.log = self.regrtest.log - self.ns = regrtest.ns + self.ns = ns self.output: queue.Queue[QueueOutput] = queue.Queue() - self.pending = MultiprocessIterator(self.regrtest.tests) - if self.ns.timeout is not None: + tests_iter = runtests.iter_tests() + self.pending = MultiprocessIterator(tests_iter) + if timeout is not None: # Rely on faulthandler to kill a worker process. This timouet is # when faulthandler fails to kill a worker process. Give a maximum # of 5 minutes to faulthandler to kill the worker. - self.worker_timeout = min(self.ns.timeout * 1.5, - self.ns.timeout + 5 * 60) + self.worker_timeout = min(timeout * 1.5, timeout + 5 * 60) else: self.worker_timeout = None self.workers = None def start_workers(self) -> None: + use_mp = self.ns.use_mp + timeout = self.ns.timeout self.workers = [TestWorkerProcess(index, self) - for index in range(1, self.ns.use_mp + 1)] + for index in range(1, use_mp + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" - if self.ns.timeout: + if timeout: msg += (" (timeout: %s, worker timeout: %s)" - % (format_duration(self.ns.timeout), + % (format_duration(timeout), format_duration(self.worker_timeout))) self.log(msg) for worker in self.workers: @@ -446,6 +508,7 @@ def stop_workers(self) -> None: worker.wait_stopped(start_time) def _get_result(self) -> QueueOutput | None: + pgo = self.ns.pgo use_faulthandler = (self.ns.timeout is not None) timeout = PROGRESS_UPDATE @@ -464,7 +527,7 @@ def _get_result(self) -> QueueOutput | None: # display progress running = get_running(self.workers) - if running and not self.ns.pgo: + if running and not pgo: self.log('running: %s' % ', '.join(running)) # all worker threads are done: consume pending results @@ -475,42 +538,46 @@ def _get_result(self) -> QueueOutput | None: def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result + pgo = self.ns.pgo text = str(result) if mp_result.err_msg: # MULTIPROCESSING_ERROR text += ' (%s)' % mp_result.err_msg - elif (result.duration >= PROGRESS_MIN_TIME and not self.ns.pgo): + elif (result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) running = get_running(self.workers) - if running and not self.ns.pgo: + if running and not pgo: text += ' -- running: %s' % ', '.join(running) self.regrtest.display_progress(self.test_index, text) def _process_result(self, item: QueueOutput) -> bool: """Returns True if test runner must stop.""" + rerun = self.runtests.rerun if item[0]: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") result = TestResult("", state=State.MULTIPROCESSING_ERROR) - self.regrtest.accumulate_result(result) - return True + self.regrtest.accumulate_result(result, rerun=rerun) + return result self.test_index += 1 mp_result = item[1] - self.regrtest.accumulate_result(mp_result.result) + result = mp_result.result + self.regrtest.accumulate_result(result, rerun=rerun) self.display_result(mp_result) if mp_result.worker_stdout: print(mp_result.worker_stdout, flush=True) - if must_stop(mp_result.result, self.ns): - return True - - return False + return result def run_tests(self) -> None: + fail_fast = self.ns.failfast + fail_env_changed = self.ns.fail_env_changed + timeout = self.ns.timeout + self.start_workers() self.test_index = 0 @@ -520,14 +587,14 @@ def run_tests(self) -> None: if item is None: break - stop = self._process_result(item) - if stop: + result = self._process_result(item) + if result.must_stop(fail_fast, fail_env_changed): break except KeyboardInterrupt: print() self.regrtest.interrupted = True finally: - if self.ns.timeout is not None: + if timeout is not None: faulthandler.cancel_dump_traceback_later() # Always ensure that all worker processes are no longer @@ -536,8 +603,8 @@ def run_tests(self) -> None: self.stop_workers() -def run_tests_multiprocess(regrtest: Regrtest) -> None: - MultiprocessTestRunner(regrtest).run_tests() +def run_tests_multiprocess(regrtest: Regrtest, runtests: RunTests) -> None: + MultiprocessTestRunner(regrtest, runtests).run_tests() class EncodeTestResult(json.JSONEncoder): @@ -552,7 +619,7 @@ def default(self, o: Any) -> dict[str, Any]: return super().default(o) -def decode_test_result(d: dict[str, Any]) -> TestResult | TestStats | dict[str, Any]: +def decode_test_result(d: dict[str, Any]) -> TestResult | dict[str, Any]: """Decode a TestResult (sub)class object from a JSON dict.""" if "__test_result__" not in d: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 89a149ec5d6b36..9a60a3d40b4c2c 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -31,7 +31,7 @@ def format_duration(seconds): return ' '.join(parts) -def removepy(names): +def strip_py_suffix(names: list[str]): if not names: return for idx, name in enumerate(names): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f28a3a2632c1c5..7bac1160fd8e0a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1189,7 +1189,6 @@ def _is_full_match_test(pattern): def set_match_tests(accept_patterns=None, ignore_patterns=None): global _match_test_func, _accept_test_patterns, _ignore_test_patterns - if accept_patterns is None: accept_patterns = () if ignore_patterns is None: diff --git a/Lib/test/support/testresult.py b/Lib/test/support/testresult.py index 14474be222dc4b..de23fdd59ded95 100644 --- a/Lib/test/support/testresult.py +++ b/Lib/test/support/testresult.py @@ -8,6 +8,7 @@ import time import traceback import unittest +from test import support class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -112,6 +113,8 @@ def addExpectedFailure(self, test, err): def addFailure(self, test, err): self._add_result(test, True, failure=self.__makeErrorDict(*err)) super().addFailure(test, err) + if support.failfast: + self.stop() def addSkip(self, test, reason): self._add_result(test, skipped=reason) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 1c02d802c0b061..eb321c4ca05f1a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -5,6 +5,7 @@ """ import contextlib +import dataclasses import glob import io import locale @@ -21,6 +22,7 @@ from test import support from test.support import os_helper, TestStats from test.libregrtest import utils, setup +from test.libregrtest.runtest import normalize_test_name if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -96,11 +98,11 @@ def test_verbose(self): ns = libregrtest._parse_args([]) self.assertEqual(ns.verbose, 0) - def test_verbose2(self): - for opt in '-w', '--verbose2': + def test_rerun(self): + for opt in '-w', '--rerun', '--verbose2': with self.subTest(opt=opt): ns = libregrtest._parse_args([opt]) - self.assertTrue(ns.verbose2) + self.assertTrue(ns.rerun) def test_verbose3(self): for opt in '-W', '--verbose3': @@ -362,6 +364,13 @@ def test_unknown_option(self): 'unrecognized arguments: --unknown-option') +@dataclasses.dataclass(slots=True) +class Rerun: + name: str + match: str | None + success: bool + + class BaseTestCase(unittest.TestCase): TEST_UNIQUE_ID = 1 TESTNAME_PREFIX = 'test_regrtest_' @@ -423,11 +432,11 @@ def parse_executed_tests(self, output): def check_executed_tests(self, output, tests, skipped=(), failed=(), env_changed=(), omitted=(), - rerun={}, run_no_tests=(), + rerun=None, run_no_tests=(), resource_denied=(), randomize=False, interrupted=False, fail_env_changed=False, - *, stats): + *, stats, forever=False, filtered=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): @@ -445,11 +454,20 @@ def check_executed_tests(self, output, tests, skipped=(), failed=(), if isinstance(stats, int): stats = TestStats(stats) + rerun_failed = [] + if rerun is not None: + failed = [rerun.name] + if not rerun.success: + rerun_failed.append(rerun.name) + executed = self.parse_executed_tests(output) + total_tests = list(tests) + if rerun is not None: + total_tests.append(rerun.name) if randomize: - self.assertEqual(set(executed), set(tests), output) + self.assertEqual(set(executed), set(total_tests), output) else: - self.assertEqual(executed, tests, output) + self.assertEqual(executed, total_tests, output) def plural(count): return 's' if count != 1 else '' @@ -465,6 +483,10 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s skipped', skipped) self.check_line(output, regex) + if resource_denied: + regex = list_regex(r'%s test%s skipped \(resource denied\)', resource_denied) + self.check_line(output, regex) + if failed: regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) @@ -478,32 +500,36 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s omitted', omitted) self.check_line(output, regex) - if rerun: - regex = list_regex('%s re-run test%s', rerun.keys()) + if rerun is not None: + regex = list_regex('%s re-run test%s', [rerun.name]) self.check_line(output, regex) - regex = LOG_PREFIX + r"Re-running failed tests in verbose mode" + regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode" + self.check_line(output, regex) + regex = fr"Re-running {rerun.name} in verbose mode" + if rerun.match: + regex = fr"{regex} \(matching: {rerun.match}\)" self.check_line(output, regex) - for name, match in rerun.items(): - regex = LOG_PREFIX + f"Re-running {name} in verbose mode \\(matching: {match}\\)" - self.check_line(output, regex) if run_no_tests: regex = list_regex('%s test%s run no tests', run_no_tests) self.check_line(output, regex) - good = (len(tests) - len(skipped) - len(failed) + good = (len(tests) - len(skipped) - len(resource_denied) - len(failed) - len(omitted) - len(env_changed) - len(run_no_tests)) if good: - regex = r'%s test%s OK\.$' % (good, plural(good)) - if not skipped and not failed and good > 1: + regex = r'%s test%s OK\.' % (good, plural(good)) + if not skipped and not failed and (rerun is None or rerun.success) and good > 1: regex = 'All %s' % regex - self.check_line(output, regex) + self.check_line(output, regex, full=True) if interrupted: self.check_line(output, 'Test suite interrupted by signal SIGINT.') # Total tests - parts = [f'run={stats.tests_run:,}'] + text = f'run={stats.tests_run:,}' + if filtered: + text = fr'{text} \(filtered\)' + parts = [text] if stats.failures: parts.append(f'failures={stats.failures:,}') if stats.skipped: @@ -512,39 +538,52 @@ def list_regex(line_format, tests): self.check_line(output, line, full=True) # Total test files - report = [f'success={good}'] - if failed: - report.append(f'failed={len(failed)}') - if env_changed: - report.append(f'env_changed={len(env_changed)}') - if skipped: - report.append(f'skipped={len(skipped)}') - if resource_denied: - report.append(f'resource_denied={len(resource_denied)}') - if rerun: - report.append(f'rerun={len(rerun)}') - if run_no_tests: - report.append(f'run_no_tests={len(run_no_tests)}') + run = len(total_tests) - len(resource_denied) + if rerun is not None: + total_failed = len(rerun_failed) + total_rerun = 1 + else: + total_failed = len(failed) + total_rerun = 0 + if interrupted: + run = 0 + text = f'run={run}' + if not forever: + text = f'{text}/{len(tests)}' + if filtered: + text = fr'{text} \(filtered\)' + report = [text] + for name, ntest in ( + ('failed', total_failed), + ('env_changed', len(env_changed)), + ('skipped', len(skipped)), + ('resource_denied', len(resource_denied)), + ('rerun', total_rerun), + ('run_no_tests', len(run_no_tests)), + ): + if ntest: + report.append(f'{name}={ntest}') line = fr'Total test files: {" ".join(report)}' self.check_line(output, line, full=True) # Result - result = [] + state = [] if failed: - result.append('FAILURE') + state.append('FAILURE') elif fail_env_changed and env_changed: - result.append('ENV CHANGED') + state.append('ENV CHANGED') if interrupted: - result.append('INTERRUPTED') - if not any((good, result, failed, interrupted, skipped, + state.append('INTERRUPTED') + if not any((good, failed, interrupted, skipped, env_changed, fail_env_changed)): - result.append("NO TESTS RAN") - elif not result: - result.append('SUCCESS') - result = ', '.join(result) - if rerun: - result = 'FAILURE then %s' % result - self.check_line(output, f'Result: {result}', full=True) + state.append("NO TESTS RAN") + elif not state: + state.append('SUCCESS') + state = ', '.join(state) + if rerun is not None: + new_state = 'SUCCESS' if rerun.success else 'FAILURE' + state = 'FAILURE then ' + new_state + self.check_line(output, f'Result: {state}', full=True) def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) @@ -563,13 +602,13 @@ def run_command(self, args, input=None, exitcode=0, **kw): stdout=subprocess.PIPE, **kw) if proc.returncode != exitcode: - msg = ("Command %s failed with exit code %s\n" + msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" "\n" "stdout:\n" "---\n" "%s\n" "---\n" - % (str(args), proc.returncode, proc.stdout)) + % (str(args), proc.returncode, exitcode, proc.stdout)) if proc.stderr: msg += ("\n" "stderr:\n" @@ -738,6 +777,40 @@ def run_tests(self, *testargs, **kw): cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] return self.run_python(cmdargs, **kw) + def test_success(self): + code = textwrap.dedent(""" + import unittest + + class PassingTests(unittest.TestCase): + def test_test1(self): + pass + + def test_test2(self): + pass + + def test_test3(self): + pass + """) + tests = [self.create_test(f'ok{i}', code=code) for i in range(1, 6)] + + output = self.run_tests(*tests) + self.check_executed_tests(output, tests, + stats=3 * len(tests)) + + def test_skip(self): + code = textwrap.dedent(""" + import unittest + raise unittest.SkipTest("nope") + """) + test_ok = self.create_test('ok') + test_skip = self.create_test('skip', code=code) + tests = [test_ok, test_skip] + + output = self.run_tests(*tests) + self.check_executed_tests(output, tests, + skipped=[test_skip], + stats=1) + def test_failing_test(self): # test a failing test code = textwrap.dedent(""" @@ -777,14 +850,12 @@ def test_pass(self): # -u audio: 1 resource enabled output = self.run_tests('-uaudio', *test_names) self.check_executed_tests(output, test_names, - skipped=tests['network'], resource_denied=tests['network'], stats=1) # no option: 0 resources enabled - output = self.run_tests(*test_names) + output = self.run_tests(*test_names, exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, test_names, - skipped=test_names, resource_denied=test_names, stats=0) @@ -930,9 +1001,21 @@ def test_run(self): builtins.__dict__['RUN'] = 1 """) test = self.create_test('forever', code=code) + + # --forever output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [test]*3, failed=test, - stats=TestStats(1, 1)) + stats=TestStats(3, 1), + forever=True) + + # --forever --rerun + output = self.run_tests('--forever', '--rerun', test, exitcode=0) + self.check_executed_tests(output, [test]*3, + rerun=Rerun(test, + match='test_run', + success=True), + stats=TestStats(4, 1), + forever=True) def check_leak(self, code, what): test = self.create_test('huntrleaks', code=code) @@ -1143,33 +1226,55 @@ def test_fail_always(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname, - rerun={testname: "test_fail_always"}, - stats=TestStats(1, 1)) + rerun=Rerun(testname, + "test_fail_always", + success=False), + stats=TestStats(3, 2)) def test_rerun_success(self): # FAILURE then SUCCESS - code = textwrap.dedent(""" - import builtins + marker_filename = os.path.abspath("regrtest_marker_filename") + self.addCleanup(os_helper.unlink, marker_filename) + self.assertFalse(os.path.exists(marker_filename)) + + code = textwrap.dedent(f""" + import os.path import unittest + marker_filename = {marker_filename!r} + class Tests(unittest.TestCase): def test_succeed(self): return def test_fail_once(self): - if not hasattr(builtins, '_test_failed'): - builtins._test_failed = True + if not os.path.exists(marker_filename): + open(marker_filename, "w").close() self.fail("bug") """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=0) + # FAILURE then SUCCESS => exit code 0 + output = self.run_tests("--rerun", testname, exitcode=0) self.check_executed_tests(output, [testname], - rerun={testname: "test_fail_once"}, - stats=1) + rerun=Rerun(testname, + match="test_fail_once", + success=True), + stats=TestStats(3, 1)) + os_helper.unlink(marker_filename) + + # with --fail-rerun, exit code EXITCODE_BAD_TEST + # on "FAILURE then SUCCESS" state. + output = self.run_tests("--rerun", "--fail-rerun", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + rerun=Rerun(testname, + match="test_fail_once", + success=True), + stats=TestStats(3, 1)) + os_helper.unlink(marker_filename) def test_rerun_setup_class_hook_failure(self): # FAILURE then FAILURE @@ -1186,10 +1291,12 @@ def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "ExampleTests"}, + rerun=Rerun(testname, + match="ExampleTests", + success=False), stats=0) def test_rerun_teardown_class_hook_failure(self): @@ -1207,11 +1314,13 @@ def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "ExampleTests"}, - stats=1) + rerun=Rerun(testname, + match="ExampleTests", + success=False), + stats=2) def test_rerun_setup_module_hook_failure(self): # FAILURE then FAILURE @@ -1227,10 +1336,12 @@ def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: testname}, + rerun=Rerun(testname, + match=None, + success=False), stats=0) def test_rerun_teardown_module_hook_failure(self): @@ -1247,11 +1358,13 @@ def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) - self.check_executed_tests(output, testname, + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], failed=[testname], - rerun={testname: testname}, - stats=1) + rerun=Rerun(testname, + match=None, + success=False), + stats=2) def test_rerun_setup_hook_failure(self): # FAILURE then FAILURE @@ -1267,11 +1380,13 @@ def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}, - stats=1) + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) def test_rerun_teardown_hook_failure(self): # FAILURE then FAILURE @@ -1287,11 +1402,13 @@ def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}, - stats=1) + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) def test_rerun_async_setup_hook_failure(self): # FAILURE then FAILURE @@ -1307,11 +1424,12 @@ async def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, - failed=[testname], - rerun={testname: "test_success"}, - stats=1) + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) def test_rerun_async_teardown_hook_failure(self): # FAILURE then FAILURE @@ -1327,11 +1445,13 @@ async def test_success(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, testname, failed=[testname], - rerun={testname: "test_success"}, - stats=1) + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) def test_no_tests_ran(self): code = textwrap.dedent(""" @@ -1347,7 +1467,7 @@ def test_bug(self): exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname], run_no_tests=testname, - stats=0) + stats=0, filtered=True) def test_no_tests_ran_skip(self): code = textwrap.dedent(""" @@ -1378,7 +1498,7 @@ def test_bug(self): exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname, testname2], run_no_tests=[testname, testname2], - stats=0) + stats=0, filtered=True) def test_no_test_ran_some_test_exist_some_not(self): code = textwrap.dedent(""" @@ -1402,7 +1522,7 @@ def test_other_bug(self): "-m", "test_other_bug", exitcode=0) self.check_executed_tests(output, [testname, testname2], run_no_tests=[testname], - stats=1) + stats=1, filtered=True) @support.cpython_only def test_uncollectable(self): @@ -1719,6 +1839,17 @@ def test_format_duration(self): self.assertEqual(utils.format_duration(3 * 3600 + 1), '3 hour 1 sec') + def test_normalize_test_name(self): + normalize = normalize_test_name + self.assertEqual(normalize('test_access (test.test_os.FileTests.test_access)'), + 'test_access') + self.assertEqual(normalize('setUpClass (test.test_os.ChownFileTests)', is_error=True), + 'ChownFileTests') + self.assertEqual(normalize('test_success (test.test_bug.ExampleTests.test_success)', is_error=True), + 'test_success') + self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) + self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-02-01-55.gh-issue-108834.iAwXzj.rst b/Misc/NEWS.d/next/Tests/2023-09-03-02-01-55.gh-issue-108834.iAwXzj.rst new file mode 100644 index 00000000000000..43b9948db0075c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-02-01-55.gh-issue-108834.iAwXzj.rst @@ -0,0 +1,6 @@ +When regrtest reruns failed tests in verbose mode (``./python -m test +--rerun``), tests are now rerun in fresh worker processes rather than being +executed in the main process. If a test does crash or is killed by a timeout, +the main process can detect and handle the killed worker process. Tests are +rerun in parallel if the ``-jN`` option is used to run tests in parallel. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-06-17-12.gh-issue-108834.fjV-CJ.rst b/Misc/NEWS.d/next/Tests/2023-09-03-06-17-12.gh-issue-108834.fjV-CJ.rst new file mode 100644 index 00000000000000..734cc66aebee15 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-06-17-12.gh-issue-108834.fjV-CJ.rst @@ -0,0 +1,2 @@ +Rename regrtest ``--verbose2`` option (``-w``) to ``--rerun``. Keep +``--verbose2`` as a deprecated alias. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-20-15-49.gh-issue-108834.Osvmhf.rst b/Misc/NEWS.d/next/Tests/2023-09-03-20-15-49.gh-issue-108834.Osvmhf.rst new file mode 100644 index 00000000000000..098861ffa30374 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-20-15-49.gh-issue-108834.Osvmhf.rst @@ -0,0 +1,3 @@ +Add ``--fail-rerun option`` option to regrtest: if a test failed when then +passed when rerun in verbose mode, exit the process with exit code 2 +(error), instead of exit code 0 (success). Patch by Victor Stinner. From 44fc378b598eb675a847d1b7c46c8f6e81313c04 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 4 Sep 2023 01:04:05 +0200 Subject: [PATCH 479/598] gh-108765: Move export code from pyport.h to exports.h (#108855) There is no API change, since pyport.h includes exports.h. --- Include/exports.h | 72 +++++++++++++++++++++++++++++++++++++++++++++++ Include/pyport.h | 71 ---------------------------------------------- 2 files changed, 72 insertions(+), 71 deletions(-) diff --git a/Include/exports.h b/Include/exports.h index 59373c39ff757c..ce601216f17156 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -1,6 +1,29 @@ #ifndef Py_EXPORTS_H #define Py_EXPORTS_H +/* Declarations for symbol visibility. + + PyAPI_FUNC(type): Declares a public Python API function and return type + PyAPI_DATA(type): Declares public Python data and its type + PyMODINIT_FUNC: A Python module init function. If these functions are + inside the Python core, they are private to the core. + If in an extension module, it may be declared with + external linkage depending on the platform. + + As a number of platforms support/require "__declspec(dllimport/dllexport)", + we support a HAVE_DECLSPEC_DLL macro to save duplication. +*/ + +/* + All windows ports, except cygwin, are handled in PC/pyconfig.h. + + Cygwin is the only other autoconf platform requiring special + linkage handling and it uses __declspec(). +*/ +#if defined(__CYGWIN__) +# define HAVE_DECLSPEC_DLL +#endif + #if defined(_WIN32) || defined(__CYGWIN__) #if defined(Py_ENABLE_SHARED) #define Py_IMPORTED_SYMBOL __declspec(dllimport) @@ -33,4 +56,53 @@ #endif #endif +/* only get special linkage if built as shared or platform is Cygwin */ +#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) +# if defined(HAVE_DECLSPEC_DLL) +# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE +# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE + /* module init functions inside the core need no external linkage */ + /* except for Cygwin to handle embedding */ +# if defined(__CYGWIN__) +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# else /* __CYGWIN__ */ +# define PyMODINIT_FUNC PyObject* +# endif /* __CYGWIN__ */ +# else /* Py_BUILD_CORE */ + /* Building an extension module, or an embedded situation */ + /* public Python functions and data are imported */ + /* Under Cygwin, auto-import functions to prevent compilation */ + /* failures similar to those described at the bottom of 4.1: */ + /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ +# if !defined(__CYGWIN__) +# define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE +# endif /* !__CYGWIN__ */ +# define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE + /* module init functions outside the core must be exported */ +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# endif /* __cplusplus */ +# endif /* Py_BUILD_CORE */ +# endif /* HAVE_DECLSPEC_DLL */ +#endif /* Py_ENABLE_SHARED */ + +/* If no external linkage macros defined by now, create defaults */ +#ifndef PyAPI_FUNC +# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE +#endif +#ifndef PyAPI_DATA +# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE +#endif +#ifndef PyMODINIT_FUNC +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# endif /* __cplusplus */ +#endif + + #endif /* Py_EXPORTS_H */ diff --git a/Include/pyport.h b/Include/pyport.h index 4a21a446b3d904..4b6858bf527df1 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -355,79 +355,8 @@ extern "C" { # define Py_NO_INLINE #endif -/* Declarations for symbol visibility. - - PyAPI_FUNC(type): Declares a public Python API function and return type - PyAPI_DATA(type): Declares public Python data and its type - PyMODINIT_FUNC: A Python module init function. If these functions are - inside the Python core, they are private to the core. - If in an extension module, it may be declared with - external linkage depending on the platform. - - As a number of platforms support/require "__declspec(dllimport/dllexport)", - we support a HAVE_DECLSPEC_DLL macro to save duplication. -*/ - -/* - All windows ports, except cygwin, are handled in PC/pyconfig.h. - - Cygwin is the only other autoconf platform requiring special - linkage handling and it uses __declspec(). -*/ -#if defined(__CYGWIN__) -# define HAVE_DECLSPEC_DLL -#endif - #include "exports.h" -/* only get special linkage if built as shared or platform is Cygwin */ -#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) -# if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE - /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ - /* Building an extension module, or an embedded situation */ - /* public Python functions and data are imported */ - /* Under Cygwin, auto-import functions to prevent compilation */ - /* failures similar to those described at the bottom of 4.1: */ - /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) -# define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE -# endif /* !__CYGWIN__ */ -# define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ -# endif /* HAVE_DECLSPEC_DLL */ -#endif /* Py_ENABLE_SHARED */ - -/* If no external linkage macros defined by now, create defaults */ -#ifndef PyAPI_FUNC -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -#endif -#ifndef PyAPI_DATA -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE -#endif -#ifndef PyMODINIT_FUNC -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX From 5a79d2ae572fed03eb9b9554b2760a34e75191a7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 4 Sep 2023 11:21:47 +0200 Subject: [PATCH 480/598] =?UTF-8?q?Revert=20"gh-46376:=20Return=20existing?= =?UTF-8?q?=20pointer=20when=20possible=20in=20ctypes=20(#1=E2=80=A6=20(#1?= =?UTF-8?q?08688)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 08447b5deb47e2a0df87fa0a0576d300e5c909b4. Revert also _ctypes.c changes of the PyDict_ContainsString() change, commit 67266266469fe0e817736227f39537182534c1a5. --- Lib/test/test_ctypes/test_keeprefs.py | 27 -------------- ...3-07-24-01-21-16.gh-issue-46376.w-xuDL.rst | 1 - Modules/_ctypes/_ctypes.c | 35 ------------------- 3 files changed, 63 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst diff --git a/Lib/test/test_ctypes/test_keeprefs.py b/Lib/test/test_ctypes/test_keeprefs.py index c6fe1de62eae7c..23b03b64b4a716 100644 --- a/Lib/test/test_ctypes/test_keeprefs.py +++ b/Lib/test/test_ctypes/test_keeprefs.py @@ -98,33 +98,6 @@ def test_p_cint(self): x = pointer(i) self.assertEqual(x._objects, {'1': i}) - def test_pp_ownership(self): - d = c_int(123) - n = c_int(456) - - p = pointer(d) - pp = pointer(p) - - self.assertIs(pp._objects['1'], p) - self.assertIs(pp._objects['0']['1'], d) - - pp.contents.contents = n - - self.assertIs(pp._objects['1'], p) - self.assertIs(pp._objects['0']['1'], n) - - self.assertIs(p._objects['1'], n) - self.assertEqual(len(p._objects), 1) - - del d - del p - - self.assertIs(pp._objects['0']['1'], n) - self.assertEqual(len(pp._objects), 2) - - del n - - self.assertEqual(len(pp._objects), 2) class PointerToStructure(unittest.TestCase): def test(self): diff --git a/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst b/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst deleted file mode 100644 index 8e8f0245b4539b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent memory leak and use-after-free when using pointers to pointers with ctypes diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ed9efcad9ab0c8..3af3a80bfb5e95 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5148,41 +5148,6 @@ Pointer_get_contents(CDataObject *self, void *closure) stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for pointer instances */ - - PyObject *keep = GetKeepedObjects(self); - if (keep != NULL) { - // check if it's a pointer to a pointer: - // pointers will have '0' key in the _objects - int ptr_probe = PyDict_ContainsString(keep, "0"); - if (ptr_probe < 0) { - return NULL; - } - if (ptr_probe) { - PyObject *item; - if (PyDict_GetItemStringRef(keep, "1", &item) < 0) { - return NULL; - } - if (item == NULL) { - PyErr_SetString(PyExc_ValueError, - "Unexpected NULL pointer in _objects"); - return NULL; - } -#ifndef NDEBUG - CDataObject *ptr2ptr = (CDataObject *)item; - // Don't construct a new object, - // return existing one instead to preserve refcount. - // Double-check that we are returning the same thing. - assert( - *(void**) self->b_ptr == ptr2ptr->b_ptr || - *(void**) self->b_value.c == ptr2ptr->b_ptr || - *(void**) self->b_ptr == ptr2ptr->b_value.c || - *(void**) self->b_value.c == ptr2ptr->b_value.c - ); -#endif - return item; - } - } - return PyCData_FromBaseObj(stgdict->proto, (PyObject *)self, 0, *(void **)self->b_ptr); From 76f3c043b6c5971d5a13fc6decf87a80ddf7ef95 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 4 Sep 2023 12:41:13 +0300 Subject: [PATCH 481/598] gh-89392: Remove test_main() in test_netrc (GH-108860) --- Lib/test/test_netrc.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index b38cb327f68ecc..81e11a293cc4c8 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,5 +1,5 @@ import netrc, os, unittest, sys, textwrap -from test.support import os_helper, run_unittest +from test.support import os_helper try: import pwd @@ -308,8 +308,6 @@ def test_security(self): self.assertEqual(nrc.hosts['foo.domain.com'], ('anonymous', '', 'pass')) -def test_main(): - return run_unittest(NetrcTestCase) if __name__ == "__main__": - test_main() + unittest.main() From d0b22f6bd84239e50b43709f98f2bb950222cfe5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 4 Sep 2023 12:41:58 +0300 Subject: [PATCH 482/598] gh-89392: Make test_pep646_syntax discoverable (GH-108861) --- Lib/test/test_pep646_syntax.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_pep646_syntax.py b/Lib/test/test_pep646_syntax.py index 12a4227e4dc959..def313da89acbd 100644 --- a/Lib/test/test_pep646_syntax.py +++ b/Lib/test/test_pep646_syntax.py @@ -1,3 +1,5 @@ +import doctest + doctests = """ Setup @@ -317,10 +319,10 @@ __test__ = {'doctests' : doctests} -def test_main(verbose=False): - from test import support - from test import test_pep646_syntax - return support.run_doctest(test_pep646_syntax, verbose) +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + if __name__ == "__main__": - test_main(verbose=True) + unittest.main() From 074ac1f72e392a576516639f650bac0519d1cb52 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 4 Sep 2023 13:04:32 +0300 Subject: [PATCH 483/598] bpo-45229: Make ElementTree tests discoverable (GH-108859) --- Lib/test/test_xml_etree.py | 66 ++++++++++-------------------------- Lib/test/test_xml_etree_c.py | 31 ++++++++++------- 2 files changed, 35 insertions(+), 62 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index da2828c3b53af0..6d413aa68a338d 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -365,6 +365,7 @@ def test_path_cache(self): from xml.etree import ElementPath elem = ET.XML(SAMPLE_XML) + ElementPath._cache.clear() for i in range(10): ET.ElementTree(elem).find('./'+str(i)) cache_len_10 = len(ElementPath._cache) for i in range(10): ET.ElementTree(elem).find('./'+str(i)) @@ -3926,8 +3927,9 @@ def test_issue14818(self): # -------------------------------------------------------------------- class NoAcceleratorTest(unittest.TestCase): - def setUp(self): - if not pyET: + @classmethod + def setUpClass(cls): + if ET is not pyET: raise unittest.SkipTest('only for the Python version') # Test that the C accelerator was not imported for pyET @@ -4192,8 +4194,7 @@ def get_option(config, option_name, default=None): # -------------------------------------------------------------------- - -def test_main(module=None): +def setUpModule(module=None): # When invoked without a module, runs the Python ET tests by loading pyET. # Otherwise, uses the given module as the ET. global pyET @@ -4205,63 +4206,30 @@ def test_main(module=None): global ET ET = module - test_classes = [ - ModuleTest, - ElementSlicingTest, - BasicElementTest, - BadElementTest, - BadElementPathTest, - ElementTreeTest, - IOTest, - ParseErrorTest, - XIncludeTest, - ElementTreeTypeTest, - ElementFindTest, - ElementIterTest, - TreeBuilderTest, - XMLParserTest, - XMLPullParserTest, - BugsTest, - KeywordArgsTest, - BoolTest, - C14NTest, - ] - - # These tests will only run for the pure-Python version that doesn't import - # _elementtree. We can't use skipUnless here, because pyET is filled in only - # after the module is loaded. - if pyET is not ET: - test_classes.extend([ - NoAcceleratorTest, - ]) + # don't interfere with subsequent tests + def cleanup(): + global ET, pyET + ET = pyET = None + unittest.addModuleCleanup(cleanup) # Provide default namespace mapping and path cache. from xml.etree import ElementPath nsmap = ET.register_namespace._namespace_map # Copy the default namespace mapping nsmap_copy = nsmap.copy() + unittest.addModuleCleanup(nsmap.update, nsmap_copy) + unittest.addModuleCleanup(nsmap.clear) + # Copy the path cache (should be empty) path_cache = ElementPath._cache + unittest.addModuleCleanup(setattr, ElementPath, "_cache", path_cache) ElementPath._cache = path_cache.copy() + # Align the Comment/PI factories. if hasattr(ET, '_set_factories'): old_factories = ET._set_factories(ET.Comment, ET.PI) - else: - old_factories = None - - try: - return support.run_unittest(*test_classes) - finally: - from xml.etree import ElementPath - # Restore mapping and path cache - nsmap.clear() - nsmap.update(nsmap_copy) - ElementPath._cache = path_cache - if old_factories is not None: - ET._set_factories(*old_factories) - # don't interfere with subsequent tests - ET = pyET = None + unittest.addModuleCleanup(ET._set_factories, *old_factories) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index fd27b575ec8dc9..3a0fc572f457ff 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -254,20 +254,25 @@ def test_element_with_children(self): self.check_sizeof(e, self.elementsize + self.extra + struct.calcsize('8P')) -def test_main(): - from test import test_xml_etree - - # Run the tests specific to the C implementation - support.run_unittest( - MiscTests, - TestAliasWorking, - TestAcceleratorImported, - SizeofTest, - ) - # Run the same test suite as the Python module - test_xml_etree.test_main(module=cET) +def install_tests(): + # Test classes should have __module__ referring to this module. + from test import test_xml_etree + for name, base in vars(test_xml_etree).items(): + if isinstance(base, type) and issubclass(base, unittest.TestCase): + class Temp(base): + pass + Temp.__name__ = Temp.__qualname__ = name + Temp.__module__ = __name__ + assert name not in globals() + globals()[name] = Temp + +install_tests() + +def setUpModule(): + from test import test_xml_etree + test_xml_etree.setUpModule(module=cET) if __name__ == '__main__': - test_main() + unittest.main() From 5a3672cb39544de72963cccde9799313bab14d07 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 4 Sep 2023 11:36:57 +0100 Subject: [PATCH 484/598] GH-108614: Remove `TIER_ONE` and `TIER_TWO` from `_PUSH_FRAME` (GH-108725) --- Include/internal/pycore_opcode_metadata.h | 6 ++-- Python/bytecodes.c | 20 +++++++----- Python/executor.c | 1 - Python/executor_cases.c.h | 20 +++++++----- Python/generated_cases.c.h | 40 +++++++++++++---------- 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index aaf3541e095468..fa4cbd9ed31c01 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1421,9 +1421,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_PUSH_FRAME] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7f398391c5dc34..fae1da31875d66 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2974,6 +2974,7 @@ dummy_func( PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { @@ -2998,18 +2999,19 @@ dummy_func( // Eventually this should be the only occurrence of this code. frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + STORE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; - #if TIER_ONE - goto start_frame; - #endif - #if TIER_TWO - ERROR_IF(_Py_EnterRecursivePy(tstate), exit_unwind); - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif } macro(CALL_BOUND_METHOD_EXACT_ARGS) = diff --git a/Python/executor.c b/Python/executor.c index 9b3262ed4165f1..ac9104223da8ff 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -109,7 +109,6 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject pop_2_error: STACK_SHRINK(1); pop_1_error: -pop_1_exit_unwind: STACK_SHRINK(1); error: // On ERROR_IF we return NULL as the frame. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e63a36d61579a4..0d5606f5a44efe 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2245,6 +2245,7 @@ PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); break; } @@ -2281,18 +2282,19 @@ // Eventually this should be the only occurrence of this code. frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + STORE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; - #if TIER_ONE - goto start_frame; - #endif - #if TIER_TWO - if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a7bf20b9d1c7f6..34cea84245f591 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3843,6 +3843,7 @@ PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -3878,18 +3879,19 @@ // Eventually this should be the only occurrence of this code. frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + STORE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; - #if TIER_ONE - goto start_frame; - #endif - #if TIER_TWO - if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } DISPATCH(); } @@ -3920,6 +3922,7 @@ PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -3955,18 +3958,19 @@ // Eventually this should be the only occurrence of this code. frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + STORE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; - #if TIER_ONE - goto start_frame; - #endif - #if TIER_TWO - if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; - stack_pointer = _PyFrame_GetStackPointer(frame); - ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - #endif + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } DISPATCH(); } From f3b6608ba2b1db6ac449f656bf439bda8d66eb9f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 4 Sep 2023 15:24:03 +0300 Subject: [PATCH 485/598] gh-89392: Fix running test_pep646_syntax as script (GH-108875) --- Lib/test/test_pep646_syntax.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_pep646_syntax.py b/Lib/test/test_pep646_syntax.py index def313da89acbd..aac089b190bc11 100644 --- a/Lib/test/test_pep646_syntax.py +++ b/Lib/test/test_pep646_syntax.py @@ -1,4 +1,5 @@ import doctest +import unittest doctests = """ From 40e52c94a27e4cd94b48e8a705914823cbb6afed Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 4 Sep 2023 15:46:00 +0200 Subject: [PATCH 486/598] gh-107902: Don't test setting suid/sgid on systems that don't support them (GH-108368) --- Lib/test/test_tarfile.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 0d6ca4315cfda4..67009a3d2e9c24 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3808,34 +3808,43 @@ def test_modes(self): arc.add('read_group_only', mode='?---r-----') arc.add('no_bits', mode='?---------') arc.add('dir/', mode='?---rwsrwt') + arc.add('dir_all_bits/', mode='?rwsrwsrwt') - # On some systems, setting the sticky bit is a no-op. - # Check if that's the case. + # On some systems, setting the uid, gid, and/or sticky bit is a no-ops. + # Check which bits we can set, so we can compare tarfile machinery to + # a simple chmod. tmp_filename = os.path.join(TEMPDIR, "tmp.file") with open(tmp_filename, 'w'): pass - os.chmod(tmp_filename, os.stat(tmp_filename).st_mode | stat.S_ISVTX) - have_sticky_files = (os.stat(tmp_filename).st_mode & stat.S_ISVTX) + new_mode = (os.stat(tmp_filename).st_mode + | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID) + os.chmod(tmp_filename, new_mode) + got_mode = os.stat(tmp_filename).st_mode + _t_file = 't' if (got_mode & stat.S_ISVTX) else 'x' + _suid_file = 's' if (got_mode & stat.S_ISUID) else 'x' + _sgid_file = 's' if (got_mode & stat.S_ISGID) else 'x' os.unlink(tmp_filename) os.mkdir(tmp_filename) - os.chmod(tmp_filename, os.stat(tmp_filename).st_mode | stat.S_ISVTX) - have_sticky_dirs = (os.stat(tmp_filename).st_mode & stat.S_ISVTX) + new_mode = (os.stat(tmp_filename).st_mode + | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID) + os.chmod(tmp_filename, new_mode) + got_mode = os.stat(tmp_filename).st_mode + _t_dir = 't' if (got_mode & stat.S_ISVTX) else 'x' + _suid_dir = 's' if (got_mode & stat.S_ISUID) else 'x' + _sgid_dir = 's' if (got_mode & stat.S_ISGID) else 'x' os.rmdir(tmp_filename) with self.check_context(arc.open(), 'fully_trusted'): - if have_sticky_files: - self.expect_file('all_bits', mode='?rwsrwsrwt') - else: - self.expect_file('all_bits', mode='?rwsrwsrwx') + self.expect_file('all_bits', + mode=f'?rw{_suid_file}rw{_sgid_file}rw{_t_file}') self.expect_file('perm_bits', mode='?rwxrwxrwx') self.expect_file('exec_group_other', mode='?rw-rwxrwx') self.expect_file('read_group_only', mode='?---r-----') self.expect_file('no_bits', mode='?---------') - if have_sticky_dirs: - self.expect_file('dir/', mode='?---rwsrwt') - else: - self.expect_file('dir/', mode='?---rwsrwx') + self.expect_file('dir/', mode=f'?---rw{_sgid_dir}rw{_t_dir}') + self.expect_file('dir_all_bits/', + mode=f'?rw{_suid_dir}rw{_sgid_dir}rw{_t_dir}') with self.check_context(arc.open(), 'tar'): self.expect_file('all_bits', mode='?rwxr-xr-x') @@ -3844,6 +3853,7 @@ def test_modes(self): self.expect_file('read_group_only', mode='?---r-----') self.expect_file('no_bits', mode='?---------') self.expect_file('dir/', mode='?---r-xr-x') + self.expect_file('dir_all_bits/', mode='?rwxr-xr-x') with self.check_context(arc.open(), 'data'): normal_dir_mode = stat.filemode(stat.S_IMODE( @@ -3854,6 +3864,7 @@ def test_modes(self): self.expect_file('read_group_only', mode='?rw-r-----') self.expect_file('no_bits', mode='?rw-------') self.expect_file('dir/', mode=normal_dir_mode) + self.expect_file('dir_all_bits/', mode=normal_dir_mode) def test_pipe(self): # Test handling of a special file From 6ead5bd6ae20b902e6c11a3c0acede22632dc0d5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 4 Sep 2023 21:31:58 +0300 Subject: [PATCH 487/598] Disable `differing_test_runners` health check (#108886) --- .github/workflows/build.yml | 2 +- Lib/test/support/hypothesis_helper.py | 5 ++++- Tools/requirements-hypothesis.txt | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Tools/requirements-hypothesis.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d37eb4447ad11a..bb568f8d3a1a12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -431,7 +431,7 @@ jobs: VENV_PYTHON=$VENV_LOC/bin/python echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt - name: 'Restore Hypothesis database' id: cache-hypothesis-database uses: actions/cache@v3 diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index da16eb50c25958..db93eea5e912e0 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -10,7 +10,10 @@ hypothesis.settings.register_profile( "slow-is-ok", deadline=None, - suppress_health_check=[hypothesis.HealthCheck.too_slow], + suppress_health_check=[ + hypothesis.HealthCheck.too_slow, + hypothesis.HealthCheck.differing_executors, + ], ) hypothesis.settings.load_profile("slow-is-ok") diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt new file mode 100644 index 00000000000000..9db2b74c87cfb0 --- /dev/null +++ b/Tools/requirements-hypothesis.txt @@ -0,0 +1,4 @@ +# Requirements file for hypothesis that +# we use to run our property-based tests in CI. + +hypothesis==6.84.0 From 572678e1f864cb042df6962848a436d84ef7a8a4 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 4 Sep 2023 14:36:16 -0600 Subject: [PATCH 488/598] CI: Bump GitHub Actions (#108879) --- .github/workflows/build.yml | 20 +++++++++---------- .github/workflows/build_msi.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/reusable-docs.yml | 6 +++--- .github/workflows/verify-ensurepip-wheels.yml | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bb568f8d3a1a12..668ae499f06f17 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} config_hash: ${{ steps.config_hash.outputs.hash }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check for source changes id: check run: | @@ -111,13 +111,13 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Restore config.cache uses: actions/cache@v3 with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH @@ -174,7 +174,7 @@ jobs: env: IncludeUwp: 'true' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build CPython run: .\PCbuild\build.bat -e -d -p Win32 - name: Display build info @@ -191,7 +191,7 @@ jobs: env: IncludeUwp: 'true' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython @@ -213,7 +213,7 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Restore config.cache uses: actions/cache@v3 with: @@ -247,7 +247,7 @@ jobs: OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -320,7 +320,7 @@ jobs: OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Restore config.cache uses: actions/cache@v3 with: @@ -368,7 +368,7 @@ jobs: OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -478,7 +478,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Restore config.cache uses: actions/cache@v3 with: diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 22f613a88aa11e..29282dffa37ec0 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -33,6 +33,6 @@ jobs: matrix: type: [x86, x64, arm64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build CPython installer run: .\Tools\msi\build.bat --doc -${{ matrix.type }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4481ea80bfd936..27b04ba1d412e3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 82714c20e968a9..fef7b02f47cdb7 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.11" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 6150b1a7d416a3..51efa54e8d1b3d 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -23,7 +23,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Set up Python' uses: actions/setup-python@v4 with: @@ -88,7 +88,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: path: ~/.cache/pip diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 17d841f1f1c54a..4a545037bf6e2b 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3' From 7855d325e638a4b7f7b40f2c35dc80de82d8fe70 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 4 Sep 2023 15:02:25 -0600 Subject: [PATCH 489/598] Link to PEP sections in What's New in 3.12 (#108878) --- Doc/whatsnew/3.12.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index fb31a07930da25..62e470aa1b7b4c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -67,7 +67,7 @@ Summary -- Release highlights New grammar features: -* :pep:`701`: Syntactic formalization of f-strings +* :ref:`whatsnew312-pep701` Interpreter improvements: @@ -75,13 +75,13 @@ Interpreter improvements: New typing features: -* :pep:`688`: Making the buffer protocol accessible in Python +* :ref:`whatsnew312-pep688` * :ref:`whatsnew312-pep692` * :ref:`whatsnew312-pep695` -* :pep:`698`: Override Decorator for Static Typing +* :ref:`whatsnew312-pep698` Important deprecations, removals or restrictions: @@ -267,6 +267,8 @@ Inlining does result in a few visible behavior changes: Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`. +.. _whatsnew312-pep688: + PEP 688: Making the buffer protocol accessible in Python -------------------------------------------------------- @@ -340,6 +342,8 @@ See :pep:`692` for more details. (Contributed by Franek Magiera in :gh:`103629`.) +.. _whatsnew312-pep698: + PEP 698: Override Decorator for Static Typing --------------------------------------------- From 6304d983a0656c1841769bf36e5b42819508d21c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 4 Sep 2023 14:44:40 -0700 Subject: [PATCH 490/598] gh-108463: Make expressions/statements work as expected in pdb (#108464) --- Doc/library/pdb.rst | 4 ++ Doc/whatsnew/3.13.rst | 4 ++ Lib/pdb.py | 3 ++ Lib/test/test_pdb.py | 40 +++++++++++++++++++ ...-08-25-00-14-34.gh-issue-108463.mQApp_.rst | 1 + 5 files changed, 52 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-25-00-14-34.gh-issue-108463.mQApp_.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 3aaac15ee5780c..002eeef4c09b5d 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -252,6 +252,10 @@ change a variable or call a function. When an exception occurs in such a statement, the exception name is printed but the debugger's state is not changed. +.. versionchanged:: 3.13 + Expressions/Statements whose prefix is a pdb command are now correctly + identified and executed. + The debugger supports :ref:`aliases `. Aliases can have parameters which allows one a certain level of adaptability to the context under examination. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5d8ecbb193f157..de23172ac7a43b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -173,6 +173,10 @@ pdb the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) +* Expressions/Statements whose prefix is a pdb command are now correctly + identified and executed. + (Contributed by Tian Gao in :gh:`108464`.) + sqlite3 ------- diff --git a/Lib/pdb.py b/Lib/pdb.py index 90f26a2eb99848..b603aca8c04ee3 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -237,6 +237,9 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, pass self.allow_kbdint = False self.nosigint = nosigint + # Consider these characters as part of the command so when the users type + # c.a or c['a'], it won't be recognized as a c(ontinue) command + self.identchars = cmd.Cmd.identchars + '=.[](),"\'+-*/%@&|<>~^' # Read ~/.pdbrc and ./.pdbrc self.rcLines = [] diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 734b5c83cdff7d..a9edd1a4d83cd9 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1957,6 +1957,46 @@ def test_pdb_multiline_statement(): (Pdb) c """ +def test_pdb_show_attribute_and_item(): + """Test for multiline statement + + >>> def test_function(): + ... n = lambda x: x + ... c = {"a": 1} + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'c["a"]', + ... 'c.get("a")', + ... 'n(1)', + ... 'j=1', + ... 'j+1', + ... 'r"a"', + ... 'next(iter([1]))', + ... 'list((0, 1))', + ... 'c' + ... ]): + ... test_function() + > (5)test_function() + -> pass + (Pdb) c["a"] + 1 + (Pdb) c.get("a") + 1 + (Pdb) n(1) + 1 + (Pdb) j=1 + (Pdb) j+1 + 2 + (Pdb) r"a" + 'a' + (Pdb) next(iter([1])) + 1 + (Pdb) list((0, 1)) + [0, 1] + (Pdb) c + """ def test_pdb_issue_20766(): """Test for reference leaks when the SIGINT handler is set. diff --git a/Misc/NEWS.d/next/Library/2023-08-25-00-14-34.gh-issue-108463.mQApp_.rst b/Misc/NEWS.d/next/Library/2023-08-25-00-14-34.gh-issue-108463.mQApp_.rst new file mode 100644 index 00000000000000..a5ab8e2f9d4b59 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-25-00-14-34.gh-issue-108463.mQApp_.rst @@ -0,0 +1 @@ +Make expressions/statements work as expected in pdb From 676593859e75d4c3543d143092b77f55389006e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Sep 2023 01:54:55 +0200 Subject: [PATCH 491/598] gh-106320: Remove private _PyErr_WriteUnraisableMsg() (#108863) Move the private _PyErr_WriteUnraisableMsg() functions to the internal C API (pycore_pyerrors.h). Move write_unraisable_exc() from _testcapi to _testinternalcapi. --- Include/cpython/pyerrors.h | 4 --- Include/internal/pycore_pyerrors.h | 5 ++++ Lib/test/audit-tests.py | 2 +- Lib/test/test_sys.py | 4 +-- Modules/_ctypes/_ctypes.c | 1 + Modules/_ctypes/callbacks.c | 6 ++--- Modules/_lsprof.c | 2 ++ Modules/_testcapi/clinic/exceptions.c.h | 34 +------------------------ Modules/_testcapi/exceptions.c | 31 ---------------------- Modules/_testinternalcapi.c | 33 +++++++++++++++++++++++- Modules/_threadmodule.c | 1 + Modules/atexitmodule.c | 3 ++- Modules/clinic/_testinternalcapi.c.h | 34 ++++++++++++++++++++++++- Modules/getpath.c | 1 + Python/compile.c | 1 + Python/perf_trampoline.c | 1 + 16 files changed, 86 insertions(+), 77 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 9633a5407f28a6..da96eec4b35aab 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -116,10 +116,6 @@ PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); -PyAPI_FUNC(void) _PyErr_WriteUnraisableMsg( - const char *err_msg, - PyObject *obj); - PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( const char *func, const char *message); diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 0f16fb894d17e1..184eb35e52b47b 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -170,6 +170,11 @@ Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, P // Export for '_zoneinfo' shared extension PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); +// Export for '_lsprof' shared extension +PyAPI_FUNC(void) _PyErr_WriteUnraisableMsg( + const char *err_msg, + PyObject *obj); + #ifdef __cplusplus } #endif diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index ad8f72f556331d..f14c2635d39390 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -289,7 +289,7 @@ def hook(event, args): def test_unraisablehook(): - from _testcapi import write_unraisable_exc + from _testinternalcapi import write_unraisable_exc def unraisablehook(hookargs): pass diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index d8b684c8a008f0..e8a99244a3a28d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1199,11 +1199,11 @@ class MyType: @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): def write_unraisable_exc(self, exc, err_msg, obj): - import _testcapi + import _testinternalcapi import types err_msg2 = f"Exception ignored {err_msg}" try: - _testcapi.write_unraisable_exc(exc, err_msg, obj) + _testinternalcapi.write_unraisable_exc(exc, err_msg, obj) return types.SimpleNamespace(exc_type=type(exc), exc_value=exc, exc_traceback=exc.__traceback__, diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 3af3a80bfb5e95..184af2132c2707 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -110,6 +110,7 @@ bytes(cdata) #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 0d8ecce009a67a..1bd8fec97179e9 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -8,9 +8,9 @@ # include #endif -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_runtime.h" // _PyRuntime -#include "pycore_global_objects.h" // _Py_ID() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() +#include "pycore_runtime.h" // _Py_ID() #include diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index e7dcb6e1713212..d23a756ace887d 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -5,7 +5,9 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pystate.h" // _PyThreadState_GET() + #include "rotatingtree.h" /************************************************************/ diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 01881534329c9d..39b5f8b91a00db 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -394,38 +394,6 @@ PyDoc_STRVAR(_testcapi_set_exception__doc__, #define _TESTCAPI_SET_EXCEPTION_METHODDEF \ {"set_exception", (PyCFunction)_testcapi_set_exception, METH_O, _testcapi_set_exception__doc__}, -PyDoc_STRVAR(_testcapi_write_unraisable_exc__doc__, -"write_unraisable_exc($module, exception, err_msg, obj, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_WRITE_UNRAISABLE_EXC_METHODDEF \ - {"write_unraisable_exc", _PyCFunction_CAST(_testcapi_write_unraisable_exc), METH_FASTCALL, _testcapi_write_unraisable_exc__doc__}, - -static PyObject * -_testcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, - PyObject *err_msg, PyObject *obj); - -static PyObject * -_testcapi_write_unraisable_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *exc; - PyObject *err_msg; - PyObject *obj; - - if (!_PyArg_CheckPositional("write_unraisable_exc", nargs, 3, 3)) { - goto exit; - } - exc = args[0]; - err_msg = args[1]; - obj = args[2]; - return_value = _testcapi_write_unraisable_exc_impl(module, exc, err_msg, obj); - -exit: - return return_value; -} - PyDoc_STRVAR(_testcapi_traceback_print__doc__, "traceback_print($module, traceback, file, /)\n" "--\n" @@ -487,4 +455,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=8f273949da28ffb5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ff19512450b3bbdb input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 025b42db247e81..b1388d75711774 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -278,36 +278,6 @@ _testcapi_set_exception(PyObject *module, PyObject *new_exc) return exc; } -/*[clinic input] -_testcapi.write_unraisable_exc - exception as exc: object - err_msg: object - obj: object - / -[clinic start generated code]*/ - -static PyObject * -_testcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, - PyObject *err_msg, PyObject *obj) -/*[clinic end generated code: output=39827c5e0a8c2092 input=582498da5b2ee6cf]*/ -{ - - const char *err_msg_utf8; - if (err_msg != Py_None) { - err_msg_utf8 = PyUnicode_AsUTF8(err_msg); - if (err_msg_utf8 == NULL) { - return NULL; - } - } - else { - err_msg_utf8 = NULL; - } - - PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); - _PyErr_WriteUnraisableMsg(err_msg_utf8, obj); - Py_RETURN_NONE; -} - /*[clinic input] _testcapi.traceback_print traceback: object @@ -384,7 +354,6 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_EXC_INFO_METHODDEF _TESTCAPI_SET_EXCEPTION_METHODDEF _TESTCAPI_TRACEBACK_PRINT_METHODDEF - _TESTCAPI_WRITE_UNRAISABLE_EXC_METHODDEF _TESTCAPI_UNSTABLE_EXC_PREP_RERAISE_STAR_METHODDEF {NULL}, }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 96d568294a9710..b6792e38fa98c2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -26,7 +26,6 @@ #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() #include "interpreteridobject.h" // PyInterpreterID_LookUp() @@ -1448,6 +1447,37 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +/*[clinic input] +_testinternalcapi.write_unraisable_exc + exception as exc: object + err_msg: object + obj: object + / +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, + PyObject *err_msg, PyObject *obj) +/*[clinic end generated code: output=a0f063cdd04aad83 input=274381b1a3fa5cd6]*/ +{ + + const char *err_msg_utf8; + if (err_msg != Py_None) { + err_msg_utf8 = PyUnicode_AsUTF8(err_msg); + if (err_msg_utf8 == NULL) { + return NULL; + } + } + else { + err_msg_utf8 = NULL; + } + + PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); + _PyErr_WriteUnraisableMsg(err_msg_utf8, obj); + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1503,6 +1533,7 @@ static PyMethodDef module_functions[] = { {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 2cf866c5a11416..49f34fcb9feb70 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -7,6 +7,7 @@ #include "pycore_dict.h" // _PyDict_Pop() #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index cec177cfc2f9c8..57e2ea67450453 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -7,9 +7,10 @@ */ #include "Python.h" -#include "pycore_atexit.h" +#include "pycore_atexit.h" // export _Py_AtExit() #include "pycore_initconfig.h" // _PyStatus_NO_MEMORY #include "pycore_interp.h" // PyInterpreterState.atexit +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pystate.h" // _PyInterpreterState_GET /* ===================================================================== */ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 28761387023508..38a3579d7dec77 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -264,4 +264,36 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=cba1c94ff4015b82 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testinternalcapi_write_unraisable_exc__doc__, +"write_unraisable_exc($module, exception, err_msg, obj, /)\n" +"--\n" +"\n"); + +#define _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF \ + {"write_unraisable_exc", _PyCFunction_CAST(_testinternalcapi_write_unraisable_exc), METH_FASTCALL, _testinternalcapi_write_unraisable_exc__doc__}, + +static PyObject * +_testinternalcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, + PyObject *err_msg, PyObject *obj); + +static PyObject * +_testinternalcapi_write_unraisable_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc; + PyObject *err_msg; + PyObject *obj; + + if (!_PyArg_CheckPositional("write_unraisable_exc", nargs, 3, 3)) { + goto exit; + } + exc = args[0]; + err_msg = args[1]; + obj = args[2]; + return_value = _testinternalcapi_write_unraisable_exc_impl(module, exc, err_msg, obj); + +exit: + return return_value; +} +/*[clinic end generated code: output=c7156622e80df1ce input=a9049054013a1b77]*/ diff --git a/Modules/getpath.c b/Modules/getpath.c index 71e23e1edaf11c..3b926cac0d3f24 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -4,6 +4,7 @@ #include "pycore_fileutils.h" // _Py_abspath() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_pathconfig.h" // _PyPathConfig_ReadGlobal() +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pymem.h" // _PyMem_RawWcsdup() #include "marshal.h" // PyMarshal_ReadObjectFromString diff --git a/Python/compile.c b/Python/compile.c index 50e29b4607f7ce..ae9edc90ea4367 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -34,6 +34,7 @@ #include "pycore_flowgraph.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_symtable.h" // PySTEntryObject, _PyFuture_FromAST() diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 10675bf9f8292a..0f1af30226d9fb 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -133,6 +133,7 @@ any DWARF information available for them). #include "pycore_ceval.h" #include "pycore_frame.h" #include "pycore_interp.h" +#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #ifdef PY_HAVE_PERF_TRAMPOLINE From 1170d5a292b46f754cd29c245a040f1602f70301 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Sep 2023 03:09:42 +0200 Subject: [PATCH 492/598] gh-108834: regrtest --fail-rerun exits with code 5 (#108896) When the --fail-rerun option is used and a test fails and then pass, regrtest now uses exit code 5 ("rerun) instead of 2 ("bad test"). --- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/test_regrtest.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 77a4090a826e06..ab03647ca5802f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -29,9 +29,10 @@ EXIT_TIMEOUT = 120.0 EXITCODE_BAD_TEST = 2 -EXITCODE_INTERRUPTED = 130 EXITCODE_ENV_CHANGED = 3 EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 class Regrtest: @@ -847,7 +848,7 @@ def get_exitcode(self): elif self.no_tests_run(): exitcode = EXITCODE_NO_TESTS_RAN elif self.rerun and self.ns.fail_rerun: - exitcode = EXITCODE_BAD_TEST + exitcode = EXITCODE_RERUN_FAIL return exitcode def action_run_tests(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index eb321c4ca05f1a..c5fb3dc9a11950 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -34,6 +34,7 @@ EXITCODE_BAD_TEST = 2 EXITCODE_ENV_CHANGED = 3 EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 EXITCODE_INTERRUPTED = 130 TEST_INTERRUPTED = textwrap.dedent(""" @@ -1265,10 +1266,10 @@ def test_fail_once(self): stats=TestStats(3, 1)) os_helper.unlink(marker_filename) - # with --fail-rerun, exit code EXITCODE_BAD_TEST + # with --fail-rerun, exit code EXITCODE_RERUN_FAIL # on "FAILURE then SUCCESS" state. output = self.run_tests("--rerun", "--fail-rerun", testname, - exitcode=EXITCODE_BAD_TEST) + exitcode=EXITCODE_RERUN_FAIL) self.check_executed_tests(output, [testname], rerun=Rerun(testname, match="test_fail_once", From 04a0830b00879efe057e3dfe75e9aa9c0caf1a26 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Sep 2023 08:36:43 +0300 Subject: [PATCH 493/598] gh-89392: Remove support of test_main() in libregrtest (GH-108876) --- Lib/test/libregrtest/runtest.py | 13 +++++-------- Lib/test/test_regrtest.py | 8 ++++---- .../2023-09-04-15-18-14.gh-issue-89392.8A4T5p.rst | 2 ++ 3 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-04-15-18-14.gh-issue-89392.8A4T5p.rst diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 6e3fab1a88318f..9cb71fbfb1d178 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -440,14 +440,11 @@ def _load_run_test(result: TestResult, ns: Namespace) -> None: test_mod = importlib.import_module(module_name) - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test runner. - test_main = getattr(test_mod, "test_main", None) - if test_main is not None: - test_func = test_main - else: - def test_func(): - return run_unittest(test_mod) + if hasattr(test_mod, "test_main"): + # https://github.com/python/cpython/issues/89392 + raise Exception("Module {result.test_name} defines test_main() which is no longer supported by regrtest") + def test_func(): + return run_unittest(test_mod) try: with save_env(ns, result.test_name): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c5fb3dc9a11950..aff5404408f8d0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1803,9 +1803,9 @@ def my_function(): 7948648 """ - def test_main(): - testmod = sys.modules[__name__] - return support.run_doctest(testmod) + def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests ''') testname = self.create_test(code=code) @@ -1814,7 +1814,7 @@ def test_main(): self.check_executed_tests(output, [testname], failed=[testname], randomize=True, - stats=TestStats(4, 2, 1)) + stats=TestStats(1, 1, 0)) class TestUtils(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Tests/2023-09-04-15-18-14.gh-issue-89392.8A4T5p.rst b/Misc/NEWS.d/next/Tests/2023-09-04-15-18-14.gh-issue-89392.8A4T5p.rst new file mode 100644 index 00000000000000..e1dea8e78cdd4e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-04-15-18-14.gh-issue-89392.8A4T5p.rst @@ -0,0 +1,2 @@ +Removed support of ``test_main()`` function in tests. They now always use +normal unittest test runner. From 5a2a04615171899885b977d77dc379bd78bac87f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 5 Sep 2023 08:03:53 +0100 Subject: [PATCH 494/598] GH-108390: Prevent non-local events being set with `sys.monitoring.set_local_events()` (GH-108420) --- Include/cpython/code.h | 17 ++- Include/internal/pycore_instruments.h | 2 +- Include/internal/pycore_interp.h | 2 +- Lib/test/test_monitoring.py | 30 +++- ...-08-13-17-18-22.gh-issue-108390.TkBccC.rst | 4 + Python/ceval.c | 32 +++-- Python/instrumentation.c | 130 +++++++++++------- 7 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-13-17-18-22.gh-issue-108390.TkBccC.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 24c5ec23590c94..45b09a1265df80 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,16 +8,21 @@ extern "C" { #endif - +/* Count of all local monitoring events */ +#define _PY_MONITORING_LOCAL_EVENTS 10 /* Count of all "real" monitoring events (not derived from other events) */ #define _PY_MONITORING_UNGROUPED_EVENTS 15 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 17 -/* Table of which tools are active for each monitored event. */ -typedef struct _Py_Monitors { +/* Tables of which tools are active for each monitored event. */ +typedef struct _Py_LocalMonitors { + uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; +} _Py_LocalMonitors; + +typedef struct _Py_GlobalMonitors { uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; -} _Py_Monitors; +} _Py_GlobalMonitors; /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG @@ -88,9 +93,9 @@ typedef struct { */ typedef struct { /* Monitoring specific to this code object */ - _Py_Monitors local_monitors; + _Py_LocalMonitors local_monitors; /* Monitoring that is active on this code object */ - _Py_Monitors active_monitors; + _Py_LocalMonitors active_monitors; /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; /* Information to support line events */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 43214aef7f7a1b..fad475c3dafb99 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -29,7 +29,7 @@ extern "C" { #define PY_MONITORING_EVENT_STOP_ITERATION 9 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) + ((ev) < _PY_MONITORING_LOCAL_EVENTS) /* Other events, mainly exceptions */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index f171c546efd53c..e0b7a325929257 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -187,7 +187,7 @@ struct _is { uint16_t optimizer_resume_threshold; uint16_t optimizer_backedge_threshold; - _Py_Monitors monitors; + _Py_GlobalMonitors monitors; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 845185be737eb2..95fe8d03b0b29a 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1218,9 +1218,11 @@ def test_instruction_then_line(self): self.check_events(self.func2, recorders = recorders, must_include = self.MUST_INCLUDE_CI) +LOCAL_RECORDERS = CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + class TestLocalEvents(MonitoringTestBase, unittest.TestCase): - def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + def check_events(self, func, expected, tool=TEST_TOOL, recorders=()): try: self.assertEqual(sys.monitoring._all_events(), {}) event_list = [] @@ -1248,7 +1250,7 @@ def func1(): line2 = 2 line3 = 3 - self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + self.check_events(func1, recorders = LOCAL_RECORDERS, expected = [ ('line', 'func1', 1), ('line', 'func1', 2), ('line', 'func1', 3)]) @@ -1260,7 +1262,7 @@ def func2(): [].append(2) line3 = 3 - self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + self.check_events(func2, recorders = LOCAL_RECORDERS, expected = [ ('line', 'func2', 1), ('line', 'func2', 2), ('call', 'append', [2]), @@ -1277,15 +1279,17 @@ def func3(): line = 5 line = 6 - self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + self.check_events(func3, recorders = LOCAL_RECORDERS, expected = [ ('line', 'func3', 1), ('line', 'func3', 2), ('line', 'func3', 3), - ('raise', KeyError), ('line', 'func3', 4), ('line', 'func3', 5), ('line', 'func3', 6)]) + def test_set_non_local_event(self): + with self.assertRaises(ValueError): + sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE) def line_from_offset(code, offset): for start, end, line in code.co_lines(): @@ -1698,3 +1702,19 @@ def run(): self.assertEqual(caught, "inner") finally: sys.monitoring.set_events(TEST_TOOL, 0) + + def test_108390(self): + + class Foo: + def __init__(self, set_event): + if set_event: + sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) + + def make_foo_optimized_then_set_event(): + for i in range(100): + Foo(i == 99) + + try: + make_foo_optimized_then_set_event() + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-13-17-18-22.gh-issue-108390.TkBccC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-13-17-18-22.gh-issue-108390.TkBccC.rst new file mode 100644 index 00000000000000..3ed596007b56f7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-13-17-18-22.gh-issue-108390.TkBccC.rst @@ -0,0 +1,4 @@ +Raise an exception when setting a non-local event (``RAISE``, ``EXCEPTION_HANDLED``, +etc.) in ``sys.monitoring.set_local_events``. + +Fixes crash when tracing in recursive calls to Python classes. diff --git a/Python/ceval.c b/Python/ceval.c index 6f90d8e6fd064b..b02bf60315b7e1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1978,28 +1978,30 @@ do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, return err; } -static inline int -no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +static inline bool +no_tools_for_global_event(PyThreadState *tstate, int event) { + return tstate->interp->monitors.tools[event] == 0; +} + +static inline bool +no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + assert(event < _PY_MONITORING_LOCAL_EVENTS); _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; if (data) { - if (data->active_monitors.tools[event] == 0) { - return 1; - } + return data->active_monitors.tools[event] == 0; } else { - if (tstate->interp->monitors.tools[event] == 0) { - return 1; - } + return no_tools_for_global_event(tstate, event); } - return 0; } static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); @@ -2009,7 +2011,7 @@ static void monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); @@ -2019,7 +2021,7 @@ static int monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { return 0; } return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); @@ -2030,7 +2032,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); @@ -2042,7 +2044,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return 0; } return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -2053,7 +2055,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 36459687be7936..9065043f55d8a7 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -138,9 +138,9 @@ is_instrumented(int opcode) #ifndef NDEBUG static inline bool -monitors_equals(_Py_Monitors a, _Py_Monitors b) +monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b) { - for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -149,42 +149,47 @@ monitors_equals(_Py_Monitors a, _Py_Monitors b) } #endif -static inline _Py_Monitors -monitors_sub(_Py_Monitors a, _Py_Monitors b) +static inline _Py_LocalMonitors +monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b) { - _Py_Monitors res; - for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + _Py_LocalMonitors res; + for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; } #ifndef NDEBUG -static inline _Py_Monitors -monitors_and(_Py_Monitors a, _Py_Monitors b) +static inline _Py_LocalMonitors +monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b) { - _Py_Monitors res; - for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + _Py_LocalMonitors res; + for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; } #endif -static inline _Py_Monitors -monitors_or(_Py_Monitors a, _Py_Monitors b) +/* The union of the *local* events in a and b. + * Global events like RAISE are ignored. + * Used for instrumentation, as only local + * events get instrumented. + */ +static inline _Py_LocalMonitors +local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) { - _Py_Monitors res; - for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + _Py_LocalMonitors res; + for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { res.tools[i] = a.tools[i] | b.tools[i]; } return res; } static inline bool -monitors_are_empty(_Py_Monitors m) +monitors_are_empty(_Py_LocalMonitors m) { - for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -193,9 +198,9 @@ monitors_are_empty(_Py_Monitors m) } static inline bool -multiple_tools(_Py_Monitors *m) +multiple_tools(_Py_LocalMonitors *m) { - for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -204,7 +209,19 @@ multiple_tools(_Py_Monitors *m) } static inline _PyMonitoringEventSet -get_events(_Py_Monitors *m, int tool_id) +get_local_events(_Py_LocalMonitors *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + +static inline _PyMonitoringEventSet +get_events(_Py_GlobalMonitors *m, int tool_id) { _PyMonitoringEventSet result = 0; for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { @@ -457,10 +474,10 @@ sanity_check_instrumentation(PyCodeObject *code) if (data == NULL) { return; } - _Py_Monitors active_monitors = _PyInterpreterState_GET()->monitors; + _Py_GlobalMonitors active_monitors = _PyInterpreterState_GET()->monitors; if (code->_co_monitoring) { _Py_Monitors local_monitors = code->_co_monitoring->local_monitors; - active_monitors = monitors_or(active_monitors, local_monitors); + active_monitors = local_union(active_monitors, local_monitors); } assert(monitors_equals( code->_co_monitoring->active_monitors, @@ -909,7 +926,7 @@ is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { - _Py_Monitors expected = monitors_or( + _Py_LocalMonitors expected = local_union( interp->monitors, code->_co_monitoring->local_monitors); return monitors_equals(code->_co_monitoring->active_monitors, expected); @@ -938,12 +955,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, } } else { - if (code->_co_monitoring) { - tools = code->_co_monitoring->active_monitors.tools[event]; - } - else { - tools = interp->monitors.tools[event]; - } + tools = interp->monitors.tools[event]; } return tools; } @@ -1471,7 +1483,7 @@ initialize_lines(PyCodeObject *code) } static void -initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) +initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) { uint8_t *line_tools = code->_co_monitoring->line_tools; assert(line_tools != NULL); @@ -1491,8 +1503,8 @@ allocate_instrumentation_data(PyCodeObject *code) PyErr_NoMemory(); return -1; } - code->_co_monitoring->local_monitors = (_Py_Monitors){ 0 }; - code->_co_monitoring->active_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->local_monitors = (_Py_LocalMonitors){ 0 }; + code->_co_monitoring->active_monitors = (_Py_LocalMonitors){ 0 }; code->_co_monitoring->tools = NULL; code->_co_monitoring->lines = NULL; code->_co_monitoring->line_tools = NULL; @@ -1509,7 +1521,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) if (allocate_instrumentation_data(code)) { return -1; } - _Py_Monitors all_events = monitors_or( + _Py_LocalMonitors all_events = local_union( interp->monitors, code->_co_monitoring->local_monitors); bool multitools = multiple_tools(&all_events); @@ -1577,14 +1589,23 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } int code_len = (int)Py_SIZE(code); + /* code->_co_firsttraceable >= code_len indicates + * that no instrumentation can be inserted. + * Exit early to avoid creating instrumentation + * data for potential statically allocated code + * objects. + * See https://github.com/python/cpython/issues/108390 */ + if (code->_co_firsttraceable >= code_len) { + return 0; + } if (update_instrumentation_data(code, interp)) { return -1; } - _Py_Monitors active_events = monitors_or( + _Py_LocalMonitors active_events = local_union( interp->monitors, code->_co_monitoring->local_monitors); - _Py_Monitors new_events; - _Py_Monitors removed_events; + _Py_LocalMonitors new_events; + _Py_LocalMonitors removed_events; bool restarted = interp->last_restart_version > code->_co_instrumentation_version; if (restarted) { @@ -1725,10 +1746,22 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { } static void -set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) +set_events(_Py_GlobalMonitors *m, int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { + uint8_t *tools = &m->tools[e]; + int active = (events >> e) & 1; + *tools &= ~(1 << tool_id); + *tools |= (active << tool_id); + } +} + +static void +set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { uint8_t *tools = &m->tools[e]; int val = (events >> e) & 1; *tools &= ~(1 << tool_id); @@ -1771,19 +1804,23 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); + assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); + if (code->_co_firsttraceable >= Py_SIZE(code)) { + PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name); + return -1; + } if (check_tool(interp, tool_id)) { return -1; } if (allocate_instrumentation_data(code)) { return -1; } - _Py_Monitors *local = &code->_co_monitoring->local_monitors; - uint32_t existing_events = get_events(local, tool_id); + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; + uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { return 0; } - set_events(local, tool_id, events); + set_local_events(local, tool_id, events); if (is_version_up_to_date(code, interp)) { /* Force instrumentation update */ code->_co_instrumentation_version = UINT64_MAX; @@ -1942,7 +1979,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return -1; } - _Py_Monitors *m = &_PyInterpreterState_GET()->monitors; + _Py_GlobalMonitors *m = &_PyInterpreterState_GET()->monitors; _PyMonitoringEventSet event_set = get_events(m, tool_id); return event_set; } @@ -2005,7 +2042,7 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, _PyMonitoringEventSet event_set = 0; _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; if (data != NULL) { - for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { if ((data->local_monitors.tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -2039,15 +2076,16 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, if (check_valid_tool(tool_id)) { return NULL; } - if (event_set < 0 || event_set >= (1 << _PY_MONITORING_EVENTS)) { - PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); - return NULL; - } if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); + return NULL; + } + if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { return NULL; } From 24e989211a50d36d7d291e43418b99d8b5ec5692 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 5 Sep 2023 09:01:30 +0100 Subject: [PATCH 495/598] Improve the GitHub issue forms (#108881) --- .github/ISSUE_TEMPLATE/bug.yml | 48 +++++++++++------------------- .github/ISSUE_TEMPLATE/crash.yml | 45 +++++++++------------------- .github/ISSUE_TEMPLATE/feature.yml | 26 ++++++++-------- 3 files changed, 44 insertions(+), 75 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 05f4f317ccc97e..21a375361bf58e 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -9,20 +9,26 @@ body: For help or advice on using Python, try one of the following options instead of opening a GitHub issue: - - Posting on [Discourse](https://discuss.python.org/c/users/7) + - Asking on [Discourse](https://discuss.python.org/c/users/7) or [Stack Overflow](https://stackoverflow.com) - Reading the [Python tutorial](https://docs.python.org/3/tutorial/) - Emailing [python-list](https://mail.python.org/mailman/listinfo/python-list) - - type: checkboxes + + Make sure to also search the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc) to check that the bug has not already been reported. + - type: textarea attributes: - label: Checklist - description: A bug in a third-party project (for example, `pip` or `requests`) should be reported to that project's issue tracker, not CPython - options: - - label: I am confident this is a bug in CPython, not a bug in a third-party project - required: false - - label: | - I have searched the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc), - and am confident this bug has not been reported before - required: false + label: "Bug description:" + description: > + Give a clear and concise description of what happened. + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), + and put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true - type: dropdown attributes: label: "CPython versions tested on:" @@ -47,23 +53,3 @@ body: - Other validations: required: false - - type: input - attributes: - label: "Output from running 'python -VV' on the command line:" - description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. - validations: - required: false - - type: textarea - attributes: - label: "A clear and concise description of the bug:" - description: > - Tell us what happened. - Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. - Put any code blocks inside triple backticks. - - value: | - ```python - # Add a code block here, if required - ``` - validations: - required: true diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 1ea84b893697a6..c14d7cf2599d4c 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -8,6 +8,20 @@ body: This form is for hard crashes of the Python interpreter, segmentation faults, failed C-level assertions, and similar. Unexpected exceptions raised from Python functions in the standard library count as bugs rather than crashes. The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. + - type: textarea + attributes: + label: What happened? + description: > + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), + and put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true - type: dropdown attributes: label: "CPython versions tested on:" @@ -38,34 +52,3 @@ body: description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. validations: required: false - - type: textarea - attributes: - label: What happened? - description: > - Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. - Put any code blocks inside triple backticks. - - value: | - ```python - # Add a code block here, if required - ``` - validations: - required: true - - type: textarea - attributes: - label: Error messages - description: > - Enter any error messages caused by the crash, including a core dump if there is one. - Feel free to leave this bit blank if it isn't relevant. - placeholder: | - Error messages should be formatted like this: - -
- Error messages/core dump - - ``` - # paste errors here, if you have any - ``` -
- validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 0200e623d2a3b0..4361ab2bf827fb 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -10,6 +10,19 @@ body: You'll need to demonstrate widespread support for your idea among the community. Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. + - type: textarea + attributes: + label: "Proposal:" + description: > + Explain your proposal, why it should be implemented, and how it would be used. + Add examples, if applicable. + Put any code blocks inside triple backticks. + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true - type: dropdown attributes: label: Has this already been discussed elsewhere? @@ -25,16 +38,3 @@ body: label: "Links to previous discussion of this feature:" validations: required: false - - type: textarea - attributes: - label: "Proposal:" - description: > - Explain your proposal, why it should be implemented, and how it would be used. - Add examples, if applicable. - Put any code blocks inside triple backticks. - value: | - ```python - # Add a code block here, if required - ``` - validations: - required: true From 230649f5383ac28793c499ea334b16ff671c3d02 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 5 Sep 2023 10:25:08 +0200 Subject: [PATCH 496/598] gh-108294: Add error handling for time.sleep audit event (GH-108363) I've also cleaned the tests up a bit to make this easier to test. --- Lib/test/audit-tests.py | 23 ++++++++++++--------- Lib/test/test_audit.py | 46 ++++++++++++++++++++++------------------- Modules/timemodule.c | 4 +++- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index f14c2635d39390..f0cedde308d53b 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -186,7 +186,7 @@ class C(A): ) -def test_open(): +def test_open(testfn): # SSLContext.load_dh_params uses _Py_fopen_obj rather than normal open() try: import ssl @@ -199,11 +199,11 @@ def test_open(): # All of them should fail with TestHook(raise_on_events={"open"}) as hook: for fn, *args in [ - (open, sys.argv[2], "r"), + (open, testfn, "r"), (open, sys.executable, "rb"), (open, 3, "wb"), - (open, sys.argv[2], "w", -1, None, None, None, False, lambda *a: 1), - (load_dh_params, sys.argv[2]), + (open, testfn, "w", -1, None, None, None, False, lambda *a: 1), + (load_dh_params, testfn), ]: if not fn: continue @@ -216,11 +216,11 @@ def test_open(): [ i for i in [ - (sys.argv[2], "r"), + (testfn, "r"), (sys.executable, "r"), (3, "w"), - (sys.argv[2], "w"), - (sys.argv[2], "rb") if load_dh_params else None, + (testfn, "w"), + (testfn, "rb") if load_dh_params else None, ] if i is not None ], @@ -517,12 +517,15 @@ def test_not_in_gc(): assert hook not in o -def test_time(): +def test_time(mode): import time def hook(event, args): if event.startswith("time."): - print(event, *args) + if mode == 'print': + print(event, *args) + elif mode == 'fail': + raise AssertionError('hook failed') sys.addaudithook(hook) time.sleep(0) @@ -549,4 +552,4 @@ def hook(event, args): suppress_msvcrt_asserts() test = sys.argv[1] - globals()[test]() + globals()[test](*sys.argv[2:]) diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 3a15835917cc32..47e5832d311bd1 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -19,7 +19,7 @@ class AuditTest(unittest.TestCase): maxDiff = None @support.requires_subprocess() - def do_test(self, *args): + def run_test_in_subprocess(self, *args): with subprocess.Popen( [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], encoding="utf-8", @@ -27,27 +27,26 @@ def do_test(self, *args): stderr=subprocess.PIPE, ) as p: p.wait() - sys.stdout.writelines(p.stdout) - sys.stderr.writelines(p.stderr) - if p.returncode: - self.fail("".join(p.stderr)) + return p, p.stdout.read(), p.stderr.read() - @support.requires_subprocess() - def run_python(self, *args): + def do_test(self, *args): + proc, stdout, stderr = self.run_test_in_subprocess(*args) + + sys.stdout.write(stdout) + sys.stderr.write(stderr) + if proc.returncode: + self.fail(stderr) + + def run_python(self, *args, expect_stderr=False): events = [] - with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], - encoding="utf-8", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as p: - p.wait() - sys.stderr.writelines(p.stderr) - return ( - p.returncode, - [line.strip().partition(" ") for line in p.stdout], - "".join(p.stderr), - ) + proc, stdout, stderr = self.run_test_in_subprocess(*args) + if not expect_stderr or support.verbose: + sys.stderr.write(stderr) + return ( + proc.returncode, + [line.strip().partition(" ") for line in stdout.splitlines()], + stderr, + ) def test_basic(self): self.do_test("test_basic") @@ -257,7 +256,7 @@ def test_not_in_gc(self): self.fail(stderr) def test_time(self): - returncode, events, stderr = self.run_python("test_time") + returncode, events, stderr = self.run_python("test_time", "print") if returncode: self.fail(stderr) @@ -271,6 +270,11 @@ def test_time(self): self.assertEqual(actual, expected) + def test_time_fail(self): + returncode, events, stderr = self.run_python("test_time", "fail", + expect_stderr=True) + self.assertNotEqual(returncode, 0) + self.assertIn('hook failed', stderr.splitlines()[-1]) def test_sys_monitoring_register_callback(self): returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") diff --git a/Modules/timemodule.c b/Modules/timemodule.c index a2b66520ee885d..6a872a285d289b 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -413,7 +413,9 @@ Return the clk_id of a thread's CPU time clock."); static PyObject * time_sleep(PyObject *self, PyObject *timeout_obj) { - PySys_Audit("time.sleep", "O", timeout_obj); + if (PySys_Audit("time.sleep", "O", timeout_obj) < 0) { + return NULL; + } _PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) From b4c8cce9a7a9f3953eedffa277a3fe071731856d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 5 Sep 2023 11:45:54 +0300 Subject: [PATCH 497/598] gh-108840: Remove unused `TestEnumTypeSubclassing` from `test_enum` (#108841) --- Lib/test/test_enum.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 11a5b425efff9a..a838b93341a608 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4688,8 +4688,6 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Huh.TWO.value, (2, 2)) self.assertEqual(Huh.THREE.value, (3, 3, 3)) -class TestEnumTypeSubclassing(unittest.TestCase): - pass expected_help_output_with_docs = """\ Help on class Color in module %s: From 8b515f60ee1dec65cb3d64f1cc1d4b32aa2f4184 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 5 Sep 2023 12:35:52 +0100 Subject: [PATCH 498/598] GH-103082: Document PEP-669: Low Impact Monitoring for CPython (GH-107772) --- Doc/library/python.rst | 1 + Doc/library/sys.monitoring.rst | 300 +++++++++++++++++++++++++++++++++ Doc/library/sys.rst | 7 + Doc/whatsnew/3.12.rst | 15 ++ 4 files changed, 323 insertions(+) create mode 100644 Doc/library/sys.monitoring.rst diff --git a/Doc/library/python.rst b/Doc/library/python.rst index f39613f572884f..610435999d9f48 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -12,6 +12,7 @@ overview: .. toctree:: sys.rst + sys.monitoring.rst sysconfig.rst builtins.rst __main__.rst diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst new file mode 100644 index 00000000000000..7b02b95fd766a7 --- /dev/null +++ b/Doc/library/sys.monitoring.rst @@ -0,0 +1,300 @@ +:mod:`sys.monitoring` --- Execution event monitoring +==================================================== + +.. module:: sys.monitoring + :synopsis: Access and control event monitoring + +----------------- + +.. note:: + + ``sys.monitoring`` is a namespace within the ``sys`` module, + not an independent module, so there is no need to + ``import sys.monitoring``, simply ``import sys`` and then use + ``sys.monitoring``. + + +This namespace provides access to the functions and constants necessary to +activate and control event monitoring. + +As programs execute, events occur that might be of interest to tools that +monitor execution. The :mod:`!sys.monitoring` namespace provides means to +receive callbacks when events of interest occur. + +The monitoring API consists of three components: + +* Tool identifiers +* Events +* Callbacks + +Tool identifiers +---------------- + +A tool identifier is an integer and associated name. +Tool identifiers are used to discourage tools from interfering with each +other and to allow multiple tools to operate at the same time. +Currently tools are completely independent and cannot be used to +monitor each other. This restriction may be lifted in the future. + +Before registering or activating events, a tool should choose an identifier. +Identifiers are integers in the range 0 to 5. + +Registering and using tools +''''''''''''''''''''''''''' + +.. function:: use_tool_id(id: int, name: str) -> None + + Must be called before ``id`` can be used. + ``id`` must be in the range 0 to 5 inclusive. + Raises a ``ValueError`` if ``id`` is in use. + +.. function:: free_tool_id(id: int) -> None + + Should be called once a tool no longer requires ``id``. + +.. function:: get_tool(id: int) -> str | None + + Returns the name of the tool if ``id`` is in use, + otherwise it returns ``None``. + ``id`` must be in the range 0 to 5 inclusive. + +All IDs are treated the same by the VM with regard to events, but the +following IDs are pre-defined to make co-operation of tools easier:: + + sys.monitoring.DEBUGGER_ID = 0 + sys.monitoring.COVERAGE_ID = 1 + sys.monitoring.PROFILER_ID = 2 + sys.monitoring.OPTIMIZER_ID = 5 + +There is no obligation to set an ID, nor is there anything preventing a tool +from using an ID even it is already in use. +However, tools are encouraged to use a unique ID and respect other tools. + +Events +------ + +The following events are supported: + +BRANCH + A conditional branch is taken (or not). +CALL + A call in Python code (event occurs before the call). +C_RAISE + Exception raised from any callable, except Python functions (event occurs after the exit). +C_RETURN + Return from any callable, except Python functions (event occurs after the return). +EXCEPTION_HANDLED + An exception is handled. +INSTRUCTION + A VM instruction is about to be executed. +JUMP + An unconditional jump in the control flow graph is made. +LINE + An instruction is about to be executed that has a different line number from the preceding instruction. +PY_RESUME + Resumption of a Python function (for generator and coroutine functions), except for throw() calls. +PY_RETURN + Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). +PY_START + Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) +PY_THROW + A Python function is resumed by a throw() call. +PY_UNWIND + Exit from a Python function during exception unwinding. +PY_YIELD + Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). +RAISE + An exception is raised, except those that cause a ``STOP_ITERATION`` event. +RERAISE + An exception is re-raised, for example at the end of a ``finally`` block. +STOP_ITERATION + An artificial ``StopIteration`` is raised; see `the STOP_ITERATION event`_. + +More events may be added in the future. + +These events are attributes of the :mod:`!sys.monitoring.events` namespace. +Each event is represented as a power-of-2 integer constant. +To define a set of events, simply bitwise or the individual events together. +For example, to specify both ``PY_RETURN`` and ``PY_START`` events, use the +expression ``PY_RETURN | PY_START``. + +Events are divided into three groups: + +Local events +'''''''''''' + +Local events are associated with normal execution of the program and happen +at clearly defined locations. All local events can be disabled. +The local events are: + +* PY_START +* PY_RESUME +* PY_RETURN +* PY_YIELD +* CALL +* LINE +* INSTRUCTION +* JUMP +* BRANCH +* STOP_ITERATION + +Ancillary events +'''''''''''''''' + +Ancillary events can be monitored like other events, but are controlled +by another event: + +* C_RAISE +* C_RETURN + +The ``C_RETURN`` and ``C_RAISE`` events are are controlled by the ``CALL`` +event. ``C_RETURN`` and ``C_RAISE`` events will only be seen if the +corresponding ``CALL`` event is being monitored. + +Other events +'''''''''''' + +Other events are not necessarily tied to a specific location in the +program and cannot be individually disabled. + +The other events that can be monitored are: + +* PY_THROW +* PY_UNWIND +* RAISE +* EXCEPTION_HANDLED + + +The STOP_ITERATION event +'''''''''''''''''''''''' + +:pep:`PEP 380 <380#use-of-stopiteration-to-return-values>` +specifies that a ``StopIteration`` exception is raised when returning a value +from a generator or coroutine. However, this is a very inefficient way to +return a value, so some Python implementations, notably CPython 3.12+, do not +raise an exception unless it would be visible to other code. + +To allow tools to monitor for real exceptions without slowing down generators +and coroutines, the ``STOP_ITERATION`` event is provided. +``STOP_ITERATION`` can be locally disabled, unlike ``RAISE``. + + +Turning events on and off +------------------------- + +In order to monitor an event, it must be turned on and a callback registered. +Events can be turned on or off by setting the events either globally or +for a particular code object. + + +Setting events globally +''''''''''''''''''''''' + +Events can be controlled globally by modifying the set of events being monitored. + +.. function:: get_events(tool_id: int) -> int + + Returns the ``int`` representing all the active events. + +.. function:: set_events(tool_id: int, event_set: int) + + Activates all events which are set in ``event_set``. + Raises a ``ValueError`` if ``tool_id`` is not in use. + +No events are active by default. + +Per code object events +'''''''''''''''''''''' + +Events can also be controlled on a per code object basis. + +.. function:: get_local_events(tool_id: int, code: CodeType) -> int + + Returns all the local events for ``code`` + +.. function:: set_local_events(tool_id: int, code: CodeType, event_set: int) + + Activates all the local events for ``code`` which are set in ``event_set``. + Raises a ``ValueError`` if ``tool_id`` is not in use. + +Local events add to global events, but do not mask them. +In other words, all global events will trigger for a code object, +regardless of the local events. + + +Disabling events +'''''''''''''''' + +Local events can be disabled for a specific code location by returning +``sys.monitoring.DISABLE`` from a callback function. This does not change +which events are set, or any other code locations for the same event. + +Disabling events for specific locations is very important for high +performance monitoring. For example, a program can be run under a +debugger with no overhead if the debugger disables all monitoring +except for a few breakpoints. + + +Registering callback functions +------------------------------ + +To register a callable for events call + +.. function:: register_callback(tool_id: int, event: int, func: Callable | None) -> Callable | None + + Registers the callable ``func`` for the ``event`` with the given ``tool_id`` + + If another callback was registered for the given ``tool_id`` and ``event``, + it is unregistered and returned. + Otherwise ``register_callback`` returns ``None``. + + +Functions can be unregistered by calling +``sys.monitoring.register_callback(tool_id, event, None)``. + +Callback functions can be registered and unregistered at any time. + +Registering or unregistering a callback function will generate a ``sys.audit`` event. + + +Callback function arguments +''''''''''''''''''''''''''' + +When an active event occurs, the registered callback function is called. +Different events will provide the callback function with different arguments, as follows: + +* ``PY_START`` and ``PY_RESUME``:: + + func(code: CodeType, instruction_offset: int) -> DISABLE | Any + +* ``PY_RETURN`` and ``PY_YIELD``: + + ``func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any`` + +* ``CALL``, ``C_RAISE`` and ``C_RETURN``: + + ``func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any`` + + If there are no arguments, ``arg0`` is set to ``MISSING``. + +* ``RAISE``, ``RERAISE``, ``EXCEPTION_HANDLED``, ``PY_UNWIND``, ``PY_THROW`` and ``STOP_ITERATION``: + + ``func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any`` + +* ``LINE``: + + ``func(code: CodeType, line_number: int) -> DISABLE | Any`` + +* ``BRANCH`` and ``JUMP``: + + ``func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any`` + + Note that the ``destination_offset`` is where the code will next execute. + For an untaken branch this will be the offset of the instruction following + the branch. + +* ``INSTRUCTION``: + + ``func(code: CodeType, instruction_offset: int) -> DISABLE | Any`` + + diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index e99ab90bd18e4d..ba8d80bccbd894 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1953,6 +1953,13 @@ always available. .. availability:: Windows. +.. data:: monitoring + :noindex: + + Namespace containing functions and constants for register callbacks + and controlling monitoring events. + See :mod:`sys.monitoring` for details. + .. data:: _xoptions A dictionary of the various implementation-specific flags passed through diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 62e470aa1b7b4c..acdfe9e8b19a4f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -73,6 +73,8 @@ Interpreter improvements: * :ref:`whatsnew312-pep684` +* :ref:`whatsnew312-pep669` + New typing features: * :ref:`whatsnew312-pep688` @@ -312,6 +314,19 @@ A Python API is anticipated for 3.13. (See :pep:`554`.) (Contributed by Eric Snow in :gh:`104210`, etc.) +.. _whatsnew312-pep669: + +PEP 669: Low impact monitoring for CPython +------------------------------------------ + +CPython 3.12 now supports the ability to monitor calls, +returns, lines, exceptions and other events using instrumentation. +This means that you only pay for what you use, providing support +for near-zero overhead debuggers and coverage tools. + +See :mod:`sys.monitoring` for details. + + New Features Related to Type Hints ================================== From 420c63621918d65ba362686b29495868f087281a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Sep 2023 15:00:28 +0300 Subject: [PATCH 499/598] Add missed "f" in an f-string (GH-108906) --- Lib/test/libregrtest/runtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 9cb71fbfb1d178..16ae04191da768 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -442,7 +442,7 @@ def _load_run_test(result: TestResult, ns: Namespace) -> None: if hasattr(test_mod, "test_main"): # https://github.com/python/cpython/issues/89392 - raise Exception("Module {result.test_name} defines test_main() which is no longer supported by regrtest") + raise Exception(f"Module {result.test_name} defines test_main() which is no longer supported by regrtest") def test_func(): return run_unittest(test_mod) From eaabaac7c099884f92428a7bb04ffa1f1d6080dd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Sep 2023 16:46:17 +0300 Subject: [PATCH 500/598] gh-89392: Use normal unittest runner in test_type_cache (GH-108911) --- Lib/test/test_type_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 24f83cd3e172c7..72587ecc11b6f3 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -58,4 +58,4 @@ class C: if __name__ == "__main__": - support.run_unittest(TypeCacheTests) + unittest.main() From f980cc19b9cafc09ef21e906871f810a1c89e62f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Sep 2023 17:35:28 +0300 Subject: [PATCH 501/598] gh-89392: Use unittest test runner for doctests in test_getopt (GH-108916) --- Lib/test/test_getopt.py | 71 +++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index c96a33b77fe272..c8b3442de4aa77 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -1,8 +1,8 @@ # test_getopt.py # David Goodger 2000-08-19 -from test.support import verbose, run_doctest from test.support.os_helper import EnvironmentVarGuard +import doctest import unittest import getopt @@ -134,48 +134,49 @@ def test_gnu_getopt(self): self.assertEqual(opts, [('-a', '')]) self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) - def test_libref_examples(self): - s = """ - Examples from the Library Reference: Doc/lib/libgetopt.tex + def test_issue4629(self): + longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) + self.assertEqual(longopts, [('--help', '')]) + longopts, shortopts = getopt.getopt(['--help=x'], '', ['help=']) + self.assertEqual(longopts, [('--help', 'x')]) + self.assertRaises(getopt.GetoptError, getopt.getopt, ['--help='], '', ['help']) - An example using only Unix style options: +def test_libref_examples(): + """ + Examples from the Library Reference: Doc/lib/libgetopt.tex + An example using only Unix style options: - >>> import getopt - >>> args = '-a -b -cfoo -d bar a1 a2'.split() - >>> args - ['-a', '-b', '-cfoo', '-d', 'bar', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'abc:d:') - >>> optlist - [('-a', ''), ('-b', ''), ('-c', 'foo'), ('-d', 'bar')] - >>> args - ['a1', 'a2'] - Using long option names is equally easy: + >>> import getopt + >>> args = '-a -b -cfoo -d bar a1 a2'.split() + >>> args + ['-a', '-b', '-cfoo', '-d', 'bar', 'a1', 'a2'] + >>> optlist, args = getopt.getopt(args, 'abc:d:') + >>> optlist + [('-a', ''), ('-b', ''), ('-c', 'foo'), ('-d', 'bar')] + >>> args + ['a1', 'a2'] + Using long option names is equally easy: - >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' - >>> args = s.split() - >>> args - ['--condition=foo', '--testing', '--output-file', 'abc.def', '-x', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'x', [ - ... 'condition=', 'output-file=', 'testing']) - >>> optlist - [('--condition', 'foo'), ('--testing', ''), ('--output-file', 'abc.def'), ('-x', '')] - >>> args - ['a1', 'a2'] - """ - import types - m = types.ModuleType("libreftest", s) - run_doctest(m, verbose) + >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' + >>> args = s.split() + >>> args + ['--condition=foo', '--testing', '--output-file', 'abc.def', '-x', 'a1', 'a2'] + >>> optlist, args = getopt.getopt(args, 'x', [ + ... 'condition=', 'output-file=', 'testing']) + >>> optlist + [('--condition', 'foo'), ('--testing', ''), ('--output-file', 'abc.def'), ('-x', '')] + >>> args + ['a1', 'a2'] + """ + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests - def test_issue4629(self): - longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) - self.assertEqual(longopts, [('--help', '')]) - longopts, shortopts = getopt.getopt(['--help=x'], '', ['help=']) - self.assertEqual(longopts, [('--help', 'x')]) - self.assertRaises(getopt.GetoptError, getopt.getopt, ['--help='], '', ['help']) if __name__ == "__main__": unittest.main() From 1e0d62793a84001e92f1c80b511d3a212b435acc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Sep 2023 17:56:30 +0300 Subject: [PATCH 502/598] gh-108416: Mark slow but not CPU bound test methods with requires_resource('walltime') (GH-108480) --- Lib/test/_test_multiprocessing.py | 3 +++ Lib/test/libregrtest/cmdline.py | 4 +++- Lib/test/test_concurrent_futures/executor.py | 1 + Lib/test/test_concurrent_futures/test_wait.py | 3 +++ Lib/test/test_eintr.py | 1 + Lib/test/test_faulthandler.py | 1 + Lib/test/test_httplib.py | 1 + Lib/test/test_imaplib.py | 5 ++++- Lib/test/test_io.py | 6 ++++++ Lib/test/test_logging.py | 1 + Lib/test/test_poll.py | 3 ++- Lib/test/test_signal.py | 1 + Lib/test/test_smtpnet.py | 1 + Lib/test/test_ssl.py | 2 ++ Lib/test/test_subprocess.py | 2 ++ Lib/test/test_urllib2net.py | 5 +++++ Lib/test/test_urllibnet.py | 2 ++ Lib/test/test_xmlrpc.py | 9 +++++++++ 18 files changed, 48 insertions(+), 3 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index bcbb4c2929d69e..2636a9cf7f5bee 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -675,6 +675,7 @@ def test_close(self): close_queue(q) + @support.requires_resource('walltime') def test_many_processes(self): if self.TYPE == 'threads': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -4991,6 +4992,7 @@ def test_wait_slow(self): def test_wait_socket_slow(self): self.test_wait_socket(True) + @support.requires_resource('walltime') def test_wait_timeout(self): from multiprocessing.connection import wait @@ -5019,6 +5021,7 @@ def signal_and_sleep(cls, sem, period): sem.release() time.sleep(period) + @support.requires_resource('walltime') def test_wait_integer(self): from multiprocessing.connection import wait diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 251fcacb1d14f7..d1a590d8c1a5b3 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -107,6 +107,8 @@ cpu - Used for certain CPU-heavy tests. + walltime - Long running but not CPU-bound tests. + subprocess Run all tests for the subprocess module. urlfetch - It is okay to download files required on testing. @@ -129,7 +131,7 @@ ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') # Other resources excluded from --use=all: # diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 36278bdd501971..1e7d4344740943 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -53,6 +53,7 @@ def test_map_exception(self): self.assertEqual(i.__next__(), (0, 1)) self.assertRaises(ZeroDivisionError, i.__next__) + @support.requires_resource('walltime') def test_map_timeout(self): results = [] try: diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index e4bea8b05aced6..3f64ca173c02f6 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -3,6 +3,7 @@ import time import unittest from concurrent import futures +from test import support from .util import ( CANCELLED_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, @@ -53,6 +54,7 @@ def test_first_completed_some_already_completed(self): finished) self.assertEqual(set([future1]), pending) + @support.requires_resource('walltime') def test_first_exception(self): future1 = self.executor.submit(mul, 2, 21) future2 = self.executor.submit(sleep_and_raise, 1.5) @@ -110,6 +112,7 @@ def test_all_completed(self): future2]), finished) self.assertEqual(set(), pending) + @support.requires_resource('walltime') def test_timeout(self): future1 = self.executor.submit(mul, 6, 7) future2 = self.executor.submit(time.sleep, 6) diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 528147802ba47e..49b15f1a2dba92 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -9,6 +9,7 @@ class EINTRTests(unittest.TestCase): @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") + @support.requires_resource('walltime') def test_all(self): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 907c2cda86cbae..3c1e8c150ae711 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -683,6 +683,7 @@ def test_dump_traceback_later_fd(self): with tempfile.TemporaryFile('wb+') as fp: self.check_dump_traceback_later(fd=fp.fileno()) + @support.requires_resource('walltime') def test_dump_traceback_later_twice(self): self.check_dump_traceback_later(loops=2) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index fe8105ee2bb3fa..676725c46ec694 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1954,6 +1954,7 @@ def test_networked_good_cert(self): h.close() self.assertIn('nginx', server_string) + @support.requires_resource('walltime') def test_networked_bad_cert(self): # We feed a "CA" cert that is unrelated to the server's cert import ssl diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 2b2f1f76d26db3..89cdca499f92a9 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -10,7 +10,7 @@ import threading import socket -from test.support import verbose, run_with_tz, run_with_locale, cpython_only +from test.support import verbose, run_with_tz, run_with_locale, cpython_only, requires_resource from test.support import hashlib_helper from test.support import threading_helper import unittest @@ -456,6 +456,7 @@ def test_simple_with_statement(self): with self.imap_class(*server.server_address): pass + @requires_resource('walltime') def test_imaplib_timeout_test(self): _, server = self._setup(SimpleIMAPHandler) addr = server.server_address[1] @@ -549,6 +550,7 @@ class NewIMAPSSLTests(NewIMAPTestsMixin, unittest.TestCase): imap_class = IMAP4_SSL server_class = SecureTCPServer + @requires_resource('walltime') def test_ssl_raises(self): ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ssl_context.verify_mode, ssl.CERT_REQUIRED) @@ -563,6 +565,7 @@ def test_ssl_raises(self): ssl_context=ssl_context) client.shutdown() + @requires_resource('walltime') def test_ssl_verified(self): ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.load_verify_locations(CAFILE) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 6976a64bf77e7d..022cf21a4709a2 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4468,10 +4468,12 @@ def run(): self.assertFalse(err.strip('.!')) @threading_helper.requires_working_threading() + @support.requires_resource('walltime') def test_daemon_threads_shutdown_stdout_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stdout') @threading_helper.requires_working_threading() + @support.requires_resource('walltime') def test_daemon_threads_shutdown_stderr_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stderr') @@ -4645,11 +4647,13 @@ def alarm_handler(sig, frame): os.close(r) @requires_alarm + @support.requires_resource('walltime') def test_interrupted_read_retry_buffered(self): self.check_interrupted_read_retry(lambda x: x.decode('latin1'), mode="rb") @requires_alarm + @support.requires_resource('walltime') def test_interrupted_read_retry_text(self): self.check_interrupted_read_retry(lambda x: x, mode="r", encoding="latin1") @@ -4723,10 +4727,12 @@ def alarm2(sig, frame): raise @requires_alarm + @support.requires_resource('walltime') def test_interrupted_write_retry_buffered(self): self.check_interrupted_write_retry(b"x", mode="wb") @requires_alarm + @support.requires_resource('walltime') def test_interrupted_write_retry_text(self): self.check_interrupted_write_retry("x", mode="w", encoding="latin1") diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index c2e8ff5d463607..2305e5162f901e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -680,6 +680,7 @@ def test_path_objects(self): support.is_emscripten, "Emscripten cannot fstat unlinked files." ) @threading_helper.requires_working_threading() + @support.requires_resource('walltime') def test_race(self): # Issue #14632 refers. def remove_loop(fname, tries): diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 02165a0244ddf4..1847ae95db9292 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -8,7 +8,7 @@ import time import unittest from test.support import ( - cpython_only, requires_subprocess, requires_working_socket + cpython_only, requires_subprocess, requires_working_socket, requires_resource ) from test.support import threading_helper from test.support.os_helper import TESTFN @@ -124,6 +124,7 @@ def fileno(self): # select(), modified to use poll() instead. @requires_subprocess() + @requires_resource('walltime') def test_poll2(self): cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 25afd6aabe0751..2a1a1ee22f43da 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -745,6 +745,7 @@ def test_siginterrupt_on(self): interrupted = self.readpipe_interrupted(True) self.assertTrue(interrupted) + @support.requires_resource('walltime') def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index 72f51cd8d81f59..2e0dc1aa276f35 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -61,6 +61,7 @@ def test_connect_default_port(self): server.ehlo() server.quit() + @support.requires_resource('walltime') def test_connect_using_sslcontext(self): context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.check_hostname = False diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 4e49dc5640d3f5..2c32fec5104c23 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2182,6 +2182,7 @@ def test_timeout_connect_ex(self): self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'Needs IPv6') + @support.requires_resource('walltime') def test_get_server_certificate_ipv6(self): with socket_helper.transient_internet('ipv6.google.com'): _test_get_server_certificate(self, 'ipv6.google.com', 443) @@ -2740,6 +2741,7 @@ def try_protocol_combo(server_protocol, client_protocol, expect_success, class ThreadedTests(unittest.TestCase): + @support.requires_resource('walltime') def test_echo(self): """Basic test of an SSL client connecting to a server""" if support.verbose: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0b9e9e16f55d7e..d95ef72b0da47a 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -269,6 +269,7 @@ def test_check_output_stdin_with_input_arg(self): self.assertIn('stdin', c.exception.args[0]) self.assertIn('input', c.exception.args[0]) + @support.requires_resource('walltime') def test_check_output_timeout(self): # check_output() function with timeout arg with self.assertRaises(subprocess.TimeoutExpired) as c: @@ -1643,6 +1644,7 @@ def test_check_output_stdin_with_input_arg(self): self.assertIn('stdin', c.exception.args[0]) self.assertIn('input', c.exception.args[0]) + @support.requires_resource('walltime') def test_check_output_timeout(self): with self.assertRaises(subprocess.TimeoutExpired) as c: cp = self.run_python(( diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index d8d882b2d33589..f0874d8d3ce463 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -133,6 +133,7 @@ def setUp(self): # XXX The rest of these tests aren't very good -- they don't check much. # They do sometimes catch some major disasters, though. + @support.requires_resource('walltime') def test_ftp(self): # Testing the same URL twice exercises the caching in CacheFTPHandler urls = [ @@ -196,6 +197,7 @@ def test_urlwithfrag(self): self.assertEqual(res.geturl(), "http://www.pythontest.net/index.html#frag") + @support.requires_resource('walltime') def test_redirect_url_withfrag(self): redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/" with socket_helper.transient_internet(redirect_url_with_frag): @@ -334,6 +336,7 @@ def test_http_timeout(self): FTP_HOST = 'ftp://www.pythontest.net/' + @support.requires_resource('walltime') def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST, timeout=None): @@ -352,6 +355,7 @@ def test_ftp_default_timeout(self): socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) + @support.requires_resource('walltime') def test_ftp_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST): @@ -363,6 +367,7 @@ def test_ftp_no_timeout(self): socket.setdefaulttimeout(None) self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) + @support.requires_resource('walltime') def test_ftp_timeout(self): with socket_helper.transient_internet(self.FTP_HOST): u = _urlopen_with_retry(self.FTP_HOST, timeout=60) diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 773101ce41f602..49a3b5afdebb2f 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -109,6 +109,7 @@ def test_getcode(self): open_url.close() self.assertEqual(code, 404) + @support.requires_resource('walltime') def test_bad_address(self): # Make sure proper exception is raised when connecting to a bogus # address. @@ -191,6 +192,7 @@ def test_header(self): logo = "http://www.pythontest.net/" + @support.requires_resource('walltime') def test_data_header(self): with self.urlretrieve(self.logo) as (file_location, fileheaders): datevalue = fileheaders.get('Date') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index edc741dbc60088..6c4b8384a3202e 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -1037,38 +1037,47 @@ def test_path2(self): self.assertEqual(p.add(6,8), 6+8) self.assertRaises(xmlrpclib.Fault, p.pow, 6, 8) + @support.requires_resource('walltime') def test_path3(self): p = xmlrpclib.ServerProxy(URL+"/is/broken") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) + @support.requires_resource('walltime') def test_invalid_path(self): p = xmlrpclib.ServerProxy(URL+"/invalid") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) + @support.requires_resource('walltime') def test_path_query_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v#frag") self.assertEqual(p.test(), "/foo?k=v#frag") + @support.requires_resource('walltime') def test_path_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo#frag") self.assertEqual(p.test(), "/foo#frag") + @support.requires_resource('walltime') def test_path_query(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v") self.assertEqual(p.test(), "/foo?k=v") + @support.requires_resource('walltime') def test_empty_path(self): p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.test(), "/RPC2") + @support.requires_resource('walltime') def test_root_path(self): p = xmlrpclib.ServerProxy(URL + "/") self.assertEqual(p.test(), "/") + @support.requires_resource('walltime') def test_empty_path_query(self): p = xmlrpclib.ServerProxy(URL + "?k=v") self.assertEqual(p.test(), "?k=v") + @support.requires_resource('walltime') def test_empty_path_fragment(self): p = xmlrpclib.ServerProxy(URL + "#frag") self.assertEqual(p.test(), "#frag") From ad1d6a1c20c7bd3e880c9af7251e6f39ff0e62a9 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 5 Sep 2023 18:11:12 +0300 Subject: [PATCH 503/598] gh-108903: Remove unneeded `lambda`s from `asyncio` (GH-108904) --- Doc/library/asyncio-protocol.rst | 4 ++-- Lib/asyncio/sslproto.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 7bc906eaafc1f2..9781bda8b27df0 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -746,7 +746,7 @@ received data, and close the connection:: loop = asyncio.get_running_loop() server = await loop.create_server( - lambda: EchoServerProtocol(), + EchoServerProtocol, '127.0.0.1', 8888) async with server: @@ -850,7 +850,7 @@ method, sends back received data:: # One protocol instance will be created to serve all # client requests. transport, protocol = await loop.create_datagram_endpoint( - lambda: EchoServerProtocol(), + EchoServerProtocol, local_addr=('127.0.0.1', 9999)) try: diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 488e17d8bccd5b..3eb65a8a08b5a0 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -539,7 +539,7 @@ def _start_handshake(self): # start handshake timeout count down self._handshake_timeout_handle = \ self._loop.call_later(self._ssl_handshake_timeout, - lambda: self._check_handshake_timeout()) + self._check_handshake_timeout) self._do_handshake() @@ -619,7 +619,7 @@ def _start_shutdown(self): self._set_state(SSLProtocolState.FLUSHING) self._shutdown_timeout_handle = self._loop.call_later( self._ssl_shutdown_timeout, - lambda: self._check_shutdown_timeout() + self._check_shutdown_timeout ) self._do_flush() @@ -758,7 +758,7 @@ def _do_read__buffered(self): else: break else: - self._loop.call_soon(lambda: self._do_read()) + self._loop.call_soon(self._do_read) except SSLAgainErrors: pass if offset > 0: From deea7c82682848b2a0db971a4dcc3a32c73a9f8c Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Tue, 5 Sep 2023 11:03:06 -0500 Subject: [PATCH 504/598] gh-107565: Update Windows build to use OpenSSL 3.0.10 (GH-108928) Also clean up some intermediate NEWS entries about previous versions. --- .../Security/2023-06-01-03-24-58.gh-issue-103142.GLWDMX.rst | 2 -- .../Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst | 1 - .../Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst | 1 + .../next/macOS/2023-05-30-23-30-46.gh-issue-103142.55lMXQ.rst | 1 - PCbuild/get_externals.bat | 4 ++-- PCbuild/python.props | 4 ++-- 6 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 Misc/NEWS.d/next/Security/2023-06-01-03-24-58.gh-issue-103142.GLWDMX.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst create mode 100644 Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-05-30-23-30-46.gh-issue-103142.55lMXQ.rst diff --git a/Misc/NEWS.d/next/Security/2023-06-01-03-24-58.gh-issue-103142.GLWDMX.rst b/Misc/NEWS.d/next/Security/2023-06-01-03-24-58.gh-issue-103142.GLWDMX.rst deleted file mode 100644 index 7e0836879e4f81..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-06-01-03-24-58.gh-issue-103142.GLWDMX.rst +++ /dev/null @@ -1,2 +0,0 @@ -The version of OpenSSL used in our binary builds has been upgraded to 1.1.1u -to address several CVEs. diff --git a/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst b/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst deleted file mode 100644 index 11f411be0f17c5..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows build to use OpenSSL 3.0.9 diff --git a/Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst b/Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst new file mode 100644 index 00000000000000..024a58299caed9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst @@ -0,0 +1 @@ +Update Windows build to use OpenSSL 3.0.10. diff --git a/Misc/NEWS.d/next/macOS/2023-05-30-23-30-46.gh-issue-103142.55lMXQ.rst b/Misc/NEWS.d/next/macOS/2023-05-30-23-30-46.gh-issue-103142.55lMXQ.rst deleted file mode 100644 index 1afd949d6a9f03..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-30-23-30-46.gh-issue-103142.55lMXQ.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 1.1.1u. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 257b360ba3506f..1d3abcc3def1fa 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.9 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.10 set libraries=%libraries% sqlite-3.42.0.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 @@ -76,7 +76,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.9 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.10 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index d3586235c82652..94faa8221eac5a 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,8 +74,8 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-3.0.9\ - $(ExternalsDir)openssl-bin-3.0.9\$(ArchName)\ + $(ExternalsDir)openssl-3.0.10\ + $(ExternalsDir)openssl-bin-3.0.10\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.2.13\ From 3f89b257639dd817a32079da2ae2c4436b8e82eb Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 5 Sep 2023 21:57:48 +0300 Subject: [PATCH 505/598] gh-108927: Fix test_import + test_importlib + test_unittest problem (#108929) --- Lib/test/test_unittest/test_discovery.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 946fa1258ea25e..004898ed431834 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -6,7 +6,6 @@ import pickle from test import support from test.support import import_helper -import test.test_importlib.util import unittest import unittest.mock @@ -826,6 +825,8 @@ def restore(): 'as dotted module names') def test_discovery_failed_discovery(self): + from test.test_importlib import util + loader = unittest.TestLoader() package = types.ModuleType('package') @@ -837,7 +838,7 @@ def _import(packagename, *args, **kwargs): # Since loader.discover() can modify sys.path, restore it when done. with import_helper.DirsOnSysPath(): # Make sure to remove 'package' from sys.modules when done. - with test.test_importlib.util.uncache('package'): + with util.uncache('package'): with self.assertRaises(TypeError) as cm: loader.discover('package') self.assertEqual(str(cm.exception), From 9bf350b0662fcf1a8b43b9293e6c8ecf3c711561 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Tue, 5 Sep 2023 14:22:27 -0500 Subject: [PATCH 506/598] gh-107755: Document the correct default value of slice step (GH-107756) Document the correct default value of slice step. --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 88a7fdfe6f0d50..d9974c6350fed1 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1631,7 +1631,7 @@ are always available. They are listed here in alphabetical order. .. class:: slice(stop) - slice(start, stop, step=1) + slice(start, stop, step=None) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to From 2c4c26c4ce4bb94200ff3c9b5a0f4c75eed96f31 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 6 Sep 2023 06:01:23 +1000 Subject: [PATCH 507/598] gh-108469: Update ast.unparse for unescaped quote support from PEP701 [3.12] (#108553) Co-authored-by: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> --- Lib/ast.py | 31 ++++++------------- Lib/test/test_tokenize.py | 2 +- Lib/test/test_unparse.py | 23 ++++++++++---- ...-09-03-04-37-52.gh-issue-108469.kusj40.rst | 3 ++ 4 files changed, 31 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst diff --git a/Lib/ast.py b/Lib/ast.py index 45b95963f81885..17ec7ff6f8bc12 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1225,17 +1225,7 @@ def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): def visit_JoinedStr(self, node): self.write("f") - if self._avoid_backslashes: - with self.buffered() as buffer: - self._write_fstring_inner(node) - return self._write_str_avoiding_backslashes("".join(buffer)) - - # If we don't need to avoid backslashes globally (i.e., we only need - # to avoid them inside FormattedValues), it's cosmetically preferred - # to use escaped whitespace. That is, it's preferred to use backslashes - # for cases like: f"{x}\n". To accomplish this, we keep track of what - # in our buffer corresponds to FormattedValues and what corresponds to - # Constant parts of the f-string, and allow escapes accordingly. + fstring_parts = [] for value in node.values: with self.buffered() as buffer: @@ -1247,11 +1237,14 @@ def visit_JoinedStr(self, node): new_fstring_parts = [] quote_types = list(_ALL_QUOTES) for value, is_constant in fstring_parts: - value, quote_types = self._str_literal_helper( - value, - quote_types=quote_types, - escape_special_whitespace=is_constant, - ) + if is_constant: + value, quote_types = self._str_literal_helper( + value, + quote_types=quote_types, + escape_special_whitespace=True, + ) + elif "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] new_fstring_parts.append(value) value = "".join(new_fstring_parts) @@ -1273,16 +1266,12 @@ def _write_fstring_inner(self, node): def visit_FormattedValue(self, node): def unparse_inner(inner): - unparser = type(self)(_avoid_backslashes=True) + unparser = type(self)() unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) with self.delimit("{", "}"): expr = unparse_inner(node.value) - if "\\" in expr: - raise ValueError( - "Unable to avoid backslash in f-string expression part" - ) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7863e27fccd972..dbefee655c377c 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1860,7 +1860,7 @@ def test_random_files(self): testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) - # TODO: Remove this once we can unparse PEP 701 syntax + # TODO: Remove this once we can untokenize PEP 701 syntax testfiles.remove(os.path.join(tempdir, "test_fstring.py")) for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'): diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index b3efb61e83049e..38c59e6d430b58 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -197,6 +197,10 @@ def test_fstrings_complicated(self): self.check_ast_roundtrip('''f"a\\r\\nb"''') self.check_ast_roundtrip('''f"\\u2028{'x'}"''') + def test_fstrings_pep701(self): + self.check_ast_roundtrip('f" something { my_dict["key"] } something else "') + self.check_ast_roundtrip('f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"') + def test_strings(self): self.check_ast_roundtrip("u'foo'") self.check_ast_roundtrip("r'foo'") @@ -378,8 +382,15 @@ def test_invalid_fstring_value(self): ) ) - def test_invalid_fstring_backslash(self): - self.check_invalid(ast.FormattedValue(value=ast.Constant(value="\\\\"))) + def test_fstring_backslash(self): + # valid since Python 3.12 + self.assertEqual(ast.unparse( + ast.FormattedValue( + value=ast.Constant(value="\\\\"), + conversion=-1, + format_spec=None, + ) + ), "{'\\\\\\\\'}") def test_invalid_yield_from(self): self.check_invalid(ast.YieldFrom(value=None)) @@ -502,11 +513,11 @@ def test_class_bases_and_keywords(self): self.check_src_roundtrip("class X(*args, **kwargs):\n pass") def test_fstrings(self): - self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') - self.check_src_roundtrip('''f"\\u2028{'x'}"''') + self.check_src_roundtrip("f'-{f'*{f'+{f'.{x}.'}+'}*'}-'") + self.check_src_roundtrip("f'\\u2028{'x'}'") self.check_src_roundtrip(r"f'{x}\n'") - self.check_src_roundtrip('''f''\'{"""\n"""}\\n''\'''') - self.check_src_roundtrip('''f''\'{f"""{x}\n"""}\\n''\'''') + self.check_src_roundtrip("f'{'\\n'}\\n'") + self.check_src_roundtrip("f'{f'{x}\\n'}\\n'") def test_docstrings(self): docstrings = ( diff --git a/Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst b/Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst new file mode 100644 index 00000000000000..ac0f682963daec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst @@ -0,0 +1,3 @@ +:func:`ast.unparse` now supports new :term:`f-string` syntax introduced in +Python 3.12. Note that the :term:`f-string` quotes are reselected for simplicity +under the new syntax. (Patch by Steven Sun) From b87263be9b82f778317c2bdf45ecd2ff23d0ba1b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 5 Sep 2023 13:58:39 -0700 Subject: [PATCH 508/598] gh-106581: Support multiple uops pushing new values (#108895) Also avoid the need for the awkward .clone() call in the argument to mgr.adjust_inverse() and mgr.adjust(). --- Lib/test/test_generated_cases.py | 30 +++++++++++++ Tools/cases_generator/stacking.py | 70 +++++++++++++++++++++++-------- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 54378fced54699..ed56f95af99417 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -532,6 +532,36 @@ def test_macro_cond_effect(self): """ self.run_cases_test(input, output) + def test_macro_push_push(self): + input = """ + op(A, (-- val1)) { + val1 = spam(); + } + op(B, (-- val2)) { + val2 = spam(); + } + macro(M) = A + B; + """ + output = """ + TARGET(M) { + PyObject *val1; + PyObject *val2; + // A + { + val1 = spam(); + } + // B + { + val2 = spam(); + } + STACK_GROW(2); + stack_pointer[-2] = val1; + stack_pointer[-1] = val2; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + if __name__ == "__main__": unittest.main() diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index dcdfc7ec054248..3021324e909100 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -261,15 +261,19 @@ def adjust_higher(self, eff: StackEffect) -> None: self.final_offset.higher(eff) def adjust(self, offset: StackOffset) -> None: - for down in offset.deep: + deep = list(offset.deep) + high = list(offset.high) + for down in deep: self.adjust_deeper(down) - for up in offset.high: + for up in high: self.adjust_higher(up) def adjust_inverse(self, offset: StackOffset) -> None: - for down in offset.deep: + deep = list(offset.deep) + high = list(offset.high) + for down in deep: self.adjust_higher(down) - for up in offset.high: + for up in high: self.adjust_deeper(up) def collect_vars(self) -> dict[str, StackEffect]: @@ -426,15 +430,26 @@ def write_components( ) if mgr.instr.name in ("_PUSH_FRAME", "_POP_FRAME"): - # Adjust stack to min_offset (input effects materialized) + # Adjust stack to min_offset. + # This means that all input effects of this instruction + # are materialized, but not its output effects. + # That's as intended, since these two are so special. out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) - # Use clone() since adjust_inverse() mutates final_offset - mgr.adjust_inverse(mgr.final_offset.clone()) + # However, for tier 2, pretend the stack is at final offset. + mgr.adjust_inverse(mgr.final_offset) + if tier == TIER_ONE: + # TODO: Check in analyzer that _{PUSH,POP}_FRAME is last. + assert ( + mgr is managers[-1] + ), f"Expected {mgr.instr.name!r} to be the last uop" + assert_no_pokes(managers) if mgr.instr.name == "SAVE_CURRENT_IP": next_instr_is_set = True if cache_offset: out.emit(f"next_instr += {cache_offset};") + if tier == TIER_ONE: + assert_no_pokes(managers) if len(parts) == 1: mgr.instr.write_body(out, 0, mgr.active_caches, tier) @@ -443,19 +458,41 @@ def write_components( mgr.instr.write_body(out, -4, mgr.active_caches, tier) if mgr is managers[-1] and not next_instr_is_set: - # TODO: Explain why this adjustment is needed. + # Adjust the stack to its final depth, *then* write the + # pokes for all preceding uops. + # Note that for array output effects we may still write + # past the stack top. out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) - # Use clone() since adjust_inverse() mutates final_offset - mgr.adjust_inverse(mgr.final_offset.clone()) + write_all_pokes(mgr.final_offset, managers, out) + return next_instr_is_set + + +def assert_no_pokes(managers: list[EffectManager]) -> None: + for mgr in managers: for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.assign( - poke.as_stack_effect(), - poke.effect, - ) + assert ( + poke.effect.name == UNUSED + ), f"Unexpected poke of {poke.effect.name} in {mgr.instr.name!r}" - return next_instr_is_set + +def write_all_pokes( + offset: StackOffset, managers: list[EffectManager], out: Formatter +) -> None: + # Emit all remaining pushes (pokes) + for m in managers: + m.adjust_inverse(offset) + write_pokes(m, out) + + +def write_pokes(mgr: EffectManager, out: Formatter) -> None: + for poke in mgr.pokes: + if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + poke.as_stack_effect(), + poke.effect, + ) def write_single_instr_for_abstract_interp(instr: Instruction, out: Formatter) -> None: @@ -478,8 +515,7 @@ def _write_components_for_abstract_interp( for mgr in managers: if mgr is managers[-1]: out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) - # Use clone() since adjust_inverse() mutates final_offset - mgr.adjust_inverse(mgr.final_offset.clone()) + mgr.adjust_inverse(mgr.final_offset) # NULL out the output stack effects for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: From cd2ef21b076b494224985e266c5f5f8b37c66618 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Sep 2023 23:59:40 +0200 Subject: [PATCH 509/598] gh-108962: Skip test_tempfile.test_flags() if not supported (#108964) Skip test_tempfile.test_flags() if chflags() fails with "OSError: [Errno 45] Operation not supported" (ex: on FreeBSD 13). --- Lib/test/test_tempfile.py | 18 +++++++++++++++++- ...3-09-05-23-00-09.gh-issue-108962.R4NwuU.rst | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-05-23-00-09.gh-issue-108962.R4NwuU.rst diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index db08fb1c7f2a42..1673507e2f7c91 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1834,9 +1834,25 @@ def test_modes(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) - @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.lchflags') + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') def test_flags(self): flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + + # skip the test if these flags are not supported (ex: FreeBSD 13) + filename = os_helper.TESTFN + try: + open(filename, "w").close() + try: + os.chflags(filename, flags) + except OSError as exc: + # "OSError: [Errno 45] Operation not supported" + self.skipTest(f"chflags() doesn't support " + f"UF_IMMUTABLE|UF_NOUNLINK: {exc}") + else: + os.chflags(filename, 0) + finally: + os_helper.unlink(filename) + d = self.do_create(recurse=3, dirs=2, files=2) with d: # Change files and directories flags recursively. diff --git a/Misc/NEWS.d/next/Tests/2023-09-05-23-00-09.gh-issue-108962.R4NwuU.rst b/Misc/NEWS.d/next/Tests/2023-09-05-23-00-09.gh-issue-108962.R4NwuU.rst new file mode 100644 index 00000000000000..380fb20b8881b2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-05-23-00-09.gh-issue-108962.R4NwuU.rst @@ -0,0 +1,3 @@ +Skip ``test_tempfile.test_flags()`` if ``chflags()`` fails with "OSError: +[Errno 45] Operation not supported" (ex: on FreeBSD 13). Patch by Victor +Stinner. From 6f8411cfd68134ccae01b0b4cb332578008a69e3 Mon Sep 17 00:00:00 2001 From: nabin2004 <107109731+nabin2004@users.noreply.github.com> Date: Wed, 6 Sep 2023 06:12:59 +0545 Subject: [PATCH 510/598] gh-108857: improve markup in inspect.Signature.replace() docs (#108862) --- Doc/library/inspect.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7884308a333020..603ac3263bb9ec 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -730,7 +730,7 @@ function. .. method:: Signature.replace(*[, parameters][, return_annotation]) - Create a new Signature instance based on the instance replace was invoked + Create a new Signature instance based on the instance :meth:`replace` was invoked on. It is possible to pass different ``parameters`` and/or ``return_annotation`` to override the corresponding properties of the base signature. To remove return_annotation from the copied Signature, pass in From 65a2bce70421197605feeed3351a4577462aae06 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 6 Sep 2023 11:52:53 +0300 Subject: [PATCH 511/598] gh-108717: Speedup `os.DirEntry.is_junction` function (#108718) --- Modules/clinic/posixmodule.c.h | 14 +++++--------- Modules/posixmodule.c | 6 ++---- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 854fd6cf5ec353..c3e7f86b3e33f1 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10695,22 +10695,18 @@ PyDoc_STRVAR(os_DirEntry_is_junction__doc__, "Return True if the entry is a junction; cached per entry."); #define OS_DIRENTRY_IS_JUNCTION_METHODDEF \ - {"is_junction", _PyCFunction_CAST(os_DirEntry_is_junction), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, os_DirEntry_is_junction__doc__}, + {"is_junction", (PyCFunction)os_DirEntry_is_junction, METH_NOARGS, os_DirEntry_is_junction__doc__}, static int -os_DirEntry_is_junction_impl(DirEntry *self, PyTypeObject *defining_class); +os_DirEntry_is_junction_impl(DirEntry *self); static PyObject * -os_DirEntry_is_junction(DirEntry *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os_DirEntry_is_junction(DirEntry *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; int _return_value; - if (nargs) { - PyErr_SetString(PyExc_TypeError, "is_junction() takes no arguments"); - goto exit; - } - _return_value = os_DirEntry_is_junction_impl(self, defining_class); + _return_value = os_DirEntry_is_junction_impl(self); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -11992,4 +11988,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=30c63cb556431cea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1dd5aa7495cd6e3a input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b4c502bef50ba9..c4340397fbe577 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14576,15 +14576,13 @@ os_DirEntry_is_symlink_impl(DirEntry *self, PyTypeObject *defining_class) /*[clinic input] os.DirEntry.is_junction -> bool - defining_class: defining_class - / Return True if the entry is a junction; cached per entry. [clinic start generated code]*/ static int -os_DirEntry_is_junction_impl(DirEntry *self, PyTypeObject *defining_class) -/*[clinic end generated code: output=7061a07b0ef2cd1f input=475cd36fb7d4723f]*/ +os_DirEntry_is_junction_impl(DirEntry *self) +/*[clinic end generated code: output=97f64d5d99eeccb5 input=4fc8e701eea118a1]*/ { #ifdef MS_WINDOWS return self->win32_lstat.st_reparse_tag == IO_REPARSE_TAG_MOUNT_POINT; From 44892b1c521c593a0ca23614ae27e96f846dd37c Mon Sep 17 00:00:00 2001 From: KH Date: Wed, 6 Sep 2023 18:06:41 +0900 Subject: [PATCH 512/598] Fix a typo in umarshal.py (#108803) Co-authored-by: Kumar Aditya --- Tools/build/umarshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index e05d93cf23c921..8198a1780b8dad 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -1,4 +1,4 @@ -# Implementat marshal.loads() in pure Python +# Implementation of marshal.loads() in pure Python import ast From 5f3433f210d25d366fcebfb40adf444c8f46cd59 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 6 Sep 2023 11:41:56 +0200 Subject: [PATCH 513/598] gh-106670: Fix Pdb handling of chained Exceptions with no stacks. (#108865) --- Lib/pdb.py | 29 ++++++++++-- Lib/test/test_pdb.py | 109 +++++++++++++++++++++++++++++++++---------- 2 files changed, 108 insertions(+), 30 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index b603aca8c04ee3..c31f608e6d9da5 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -494,6 +494,8 @@ def interaction(self, frame, tb_or_exc): Pdb._previous_sigint_handler = None _chained_exceptions, tb = self._get_tb_and_exceptions(tb_or_exc) + if isinstance(tb_or_exc, BaseException): + assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): if self.setup(frame, tb): # no interaction desired at this time (happens if .pdbrc contains @@ -1169,7 +1171,12 @@ def do_exceptions(self, arg): rep = repr(exc) if len(rep) > 80: rep = rep[:77] + "..." - self.message(f"{prompt} {ix:>3} {rep}") + indicator = ( + " -" + if self._chained_exceptions[ix].__traceback__ is None + else f"{ix:>3}" + ) + self.message(f"{prompt} {indicator} {rep}") else: try: number = int(arg) @@ -1177,6 +1184,10 @@ def do_exceptions(self, arg): self.error("Argument must be an integer") return if 0 <= number < len(self._chained_exceptions): + if self._chained_exceptions[number].__traceback__ is None: + self.error("This exception does not have a traceback, cannot jump to it") + return + self._chained_exception_index = number self.setup(None, self._chained_exceptions[number].__traceback__) self.print_stack_entry(self.stack[self.curindex]) @@ -2010,19 +2021,27 @@ def post_mortem(t=None): If `t` is an exception object, the `exceptions` command makes it possible to list and inspect its chained exceptions (if any). """ + return _post_mortem(t, Pdb()) + + +def _post_mortem(t, pdb_instance): + """ + Private version of post_mortem, which allow to pass a pdb instance + for testing purposes. + """ # handling the default if t is None: exc = sys.exception() if exc is not None: t = exc.__traceback__ - if t is None: + if t is None or (isinstance(t, BaseException) and t.__traceback__ is None): raise ValueError("A valid traceback must be passed if no " "exception is being handled") - p = Pdb() - p.reset() - p.interaction(None, t) + pdb_instance.reset() + pdb_instance.interaction(None, t) + def pm(): """Enter post-mortem debugging of the traceback found in sys.last_exc.""" diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index a9edd1a4d83cd9..f6bed84808ed7f 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -848,9 +848,7 @@ def test_post_mortem_chained(): ... try: ... test_function_reraise() ... except Exception as e: - ... # same as pdb.post_mortem(e), but with custom pdb instance. - ... instance.reset() - ... instance.interaction(None, e) + ... pdb._post_mortem(e, instance) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... 'exceptions', @@ -907,11 +905,18 @@ def test_post_mortem_chained(): def test_post_mortem_cause_no_context(): """Test post mortem traceback debugging of chained exception + >>> def make_exc_with_stack(type_, *content, from_=None): + ... try: + ... raise type_(*content) from from_ + ... except Exception as out: + ... return out + ... + >>> def main(): ... try: ... raise ValueError('Context Not Shown') ... except Exception as e1: - ... raise ValueError("With Cause") from TypeError('The Cause') + ... raise ValueError("With Cause") from make_exc_with_stack(TypeError,'The Cause') >>> def test_function(): ... import pdb; @@ -919,12 +924,11 @@ def test_post_mortem_cause_no_context(): ... try: ... main() ... except Exception as e: - ... # same as pdb.post_mortem(e), but with custom pdb instance. - ... instance.reset() - ... instance.interaction(None, e) + ... pdb._post_mortem(e, instance) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... 'exceptions', + ... 'exceptions 0', ... 'exceptions 1', ... 'up', ... 'down', @@ -934,20 +938,23 @@ def test_post_mortem_cause_no_context(): ... test_function() ... except ValueError: ... print('Ok.') - > (5)main() - -> raise ValueError("With Cause") from TypeError('The Cause') + > (5)main() + -> raise ValueError("With Cause") from make_exc_with_stack(TypeError,'The Cause') (Pdb) exceptions - 0 TypeError('The Cause') - > 1 ValueError('With Cause') + 0 TypeError('The Cause') + > 1 ValueError('With Cause') + (Pdb) exceptions 0 + > (3)make_exc_with_stack() + -> raise type_(*content) from from_ (Pdb) exceptions 1 - > (5)main() - -> raise ValueError("With Cause") from TypeError('The Cause') + > (5)main() + -> raise ValueError("With Cause") from make_exc_with_stack(TypeError,'The Cause') (Pdb) up - > (5)test_function() + > (5)test_function() -> main() (Pdb) down - > (5)main() - -> raise ValueError("With Cause") from TypeError('The Cause') + > (5)main() + -> raise ValueError("With Cause") from make_exc_with_stack(TypeError,'The Cause') (Pdb) exit""" @@ -971,9 +978,7 @@ def test_post_mortem_context_of_the_cause(): ... try: ... main() ... except Exception as e: - ... # same as pdb.post_mortem(e), but with custom pdb instance. - ... instance.reset() - ... instance.interaction(None, e) + ... pdb._post_mortem(e, instance) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... 'exceptions', @@ -1046,9 +1051,7 @@ def test_post_mortem_from_none(): ... try: ... main() ... except Exception as e: - ... # same as pdb.post_mortem(e), but with custom pdb instance. - ... instance.reset() - ... instance.interaction(None, e) + ... pdb._post_mortem(e, instance) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... 'exceptions', @@ -1066,6 +1069,64 @@ def test_post_mortem_from_none(): """ +def test_post_mortem_from_no_stack(): + """Test post mortem traceback debugging of chained exception + + especially when one exception has no stack. + + >>> def main(): + ... raise Exception() from Exception() + + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... try: + ... main() + ... except Exception as e: + ... pdb._post_mortem(e, instance) + + >>> with PdbTestInput( # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... ["exceptions", + ... "exceptions 0", + ... "exit"], + ... ): + ... try: + ... test_function() + ... except ValueError: + ... print('Correctly reraised.') + > (2)main() + -> raise Exception() from Exception() + (Pdb) exceptions + - Exception() + > 1 Exception() + (Pdb) exceptions 0 + *** This exception does not have a traceback, cannot jump to it + (Pdb) exit + """ + + +def test_post_mortem_single_no_stack(): + """Test post mortem called when origin exception has no stack + + + >>> def test_function(): + ... import pdb; + ... instance = pdb.Pdb(nosigint=True, readrc=False) + ... import sys + ... sys.last_exc = Exception() + ... pdb._post_mortem(sys.last_exc, instance) + + >>> with PdbTestInput( # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... [] + ... ): + ... try: + ... test_function() + ... except ValueError as e: + ... print(e) + A valid traceback must be passed if no exception is being handled + """ + def test_post_mortem_complex(): """Test post mortem traceback debugging of chained exception @@ -1130,9 +1191,7 @@ def test_post_mortem_complex(): ... try: ... main() ... except Exception as e: - ... # same as pdb.post_mortem(e), but with custom pdb instance. - ... instance.reset() - ... instance.interaction(None, e) + ... pdb._post_mortem(e, instance) >>> with PdbTestInput( # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... ["exceptions", From 39376cb93d40b8fe588be0c1987272b0f8c49e26 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 6 Sep 2023 12:54:59 +0100 Subject: [PATCH 514/598] gh-108991: replace _PyFrame_GetState by two simpler functions (#108992) --- Objects/frameobject.c | 51 +++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 28f5a5a1222806..7017cc6c0e295f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -591,39 +591,28 @@ first_line_not_before(int *lines, int len, int line) return result; } -static PyFrameState -_PyFrame_GetState(PyFrameObject *frame) +static bool +frame_is_cleared(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); if (frame->f_frame->stacktop == 0) { - return FRAME_CLEARED; + return true; } - switch(frame->f_frame->owner) { - case FRAME_OWNED_BY_GENERATOR: - { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state; - } - case FRAME_OWNED_BY_THREAD: - { - if (_PyInterpreterFrame_LASTI(frame->f_frame) < 0) { - return FRAME_CREATED; - } - switch (frame->f_frame->prev_instr->op.code) - { - case COPY_FREE_VARS: - case MAKE_CELL: - case RETURN_GENERATOR: - /* Frame not fully initialized */ - return FRAME_CREATED; - default: - return FRAME_EXECUTING; - } - } - case FRAME_OWNED_BY_FRAME_OBJECT: - return FRAME_COMPLETED; + if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + return gen->gi_frame_state == FRAME_CLEARED; + } + return false; +} + +static bool frame_is_suspended(PyFrameObject *frame) +{ + assert(!_PyFrame_IsIncomplete(frame->f_frame)); + if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + return gen->gi_frame_state == FRAME_SUSPENDED; } - Py_UNREACHABLE(); + return false; } /* Setter for f_lineno - you can set f_lineno from within a trace function in @@ -655,7 +644,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } - PyFrameState state = _PyFrame_GetState(f); + bool is_suspended = frame_is_suspended(f); /* * This code preserves the historical restrictions on * setting the line number of a frame. @@ -811,7 +800,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } assert(unbound == 0); } - if (state == FRAME_SUSPENDED) { + if (is_suspended) { /* Account for value popped by yield */ start_stack = pop_value(start_stack); } @@ -1455,7 +1444,7 @@ void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && _PyFrame_GetState(f) != FRAME_CLEARED) { + if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { _PyFrame_LocalsToFast(f->f_frame, clear); f->f_fast_as_locals = 0; } From 1fb20d42c58924e2e941622b3539645c7b843e0e Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 6 Sep 2023 16:41:38 +0300 Subject: [PATCH 515/598] gh-108983: Add more PEP 526 tests to `test_grammar` (#108984) --- Lib/test/test_grammar.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8507a07e498532..7c15a23a69163b 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -350,6 +350,11 @@ def test_var_annot_syntax_errors(self): check_syntax_error(self, "x: int: str") check_syntax_error(self, "def f():\n" " nonlocal x: int\n") + check_syntax_error(self, "def f():\n" + " global x: int\n") + check_syntax_error(self, "x: int = y = 1") + check_syntax_error(self, "z = w: int = 1") + check_syntax_error(self, "x: int = y: int = 1") # AST pass check_syntax_error(self, "[x, 0]: int\n") check_syntax_error(self, "f(): int\n") @@ -363,6 +368,12 @@ def test_var_annot_syntax_errors(self): check_syntax_error(self, "def f():\n" " global x\n" " x: int\n") + check_syntax_error(self, "def f():\n" + " x: int\n" + " nonlocal x\n") + check_syntax_error(self, "def f():\n" + " nonlocal x\n" + " x: int\n") def test_var_annot_basic_semantics(self): # execution order From 14d6e197cc56e5256d501839a4e66e3864ab15f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 15:54:16 +0200 Subject: [PATCH 516/598] gh-108303: Create Lib/test/test_dataclasses/ directory (#108978) Move test_dataclasses.py and its "dataclass_*.py" modules into the new Lib/test/test_dataclasses/ subdirectory. --- .../__init__.py} | 10 +++++----- Lib/test/{ => test_dataclasses}/dataclass_module_1.py | 0 .../{ => test_dataclasses}/dataclass_module_1_str.py | 0 Lib/test/{ => test_dataclasses}/dataclass_module_2.py | 0 .../{ => test_dataclasses}/dataclass_module_2_str.py | 0 Lib/test/{ => test_dataclasses}/dataclass_textanno.py | 0 Makefile.pre.in | 1 + 7 files changed, 6 insertions(+), 5 deletions(-) rename Lib/test/{test_dataclasses.py => test_dataclasses/__init__.py} (99%) rename Lib/test/{ => test_dataclasses}/dataclass_module_1.py (100%) rename Lib/test/{ => test_dataclasses}/dataclass_module_1_str.py (100%) rename Lib/test/{ => test_dataclasses}/dataclass_module_2.py (100%) rename Lib/test/{ => test_dataclasses}/dataclass_module_2_str.py (100%) rename Lib/test/{ => test_dataclasses}/dataclass_textanno.py (100%) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses/__init__.py similarity index 99% rename from Lib/test/test_dataclasses.py rename to Lib/test/test_dataclasses/__init__.py index bd8d82438414e6..7c07dfc77de208 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3684,10 +3684,10 @@ class C: self.assertEqual(C(10).x, 10) def test_classvar_module_level_import(self): - from test import dataclass_module_1 - from test import dataclass_module_1_str - from test import dataclass_module_2 - from test import dataclass_module_2_str + from test.test_dataclasses import dataclass_module_1 + from test.test_dataclasses import dataclass_module_1_str + from test.test_dataclasses import dataclass_module_2 + from test.test_dataclasses import dataclass_module_2_str for m in (dataclass_module_1, dataclass_module_1_str, dataclass_module_2, dataclass_module_2_str, @@ -3725,7 +3725,7 @@ def test_classvar_module_level_import(self): self.assertNotIn('not_iv4', c.__dict__) def test_text_annotations(self): - from test import dataclass_textanno + from test.test_dataclasses import dataclass_textanno self.assertEqual( get_type_hints(dataclass_textanno.Bar), diff --git a/Lib/test/dataclass_module_1.py b/Lib/test/test_dataclasses/dataclass_module_1.py similarity index 100% rename from Lib/test/dataclass_module_1.py rename to Lib/test/test_dataclasses/dataclass_module_1.py diff --git a/Lib/test/dataclass_module_1_str.py b/Lib/test/test_dataclasses/dataclass_module_1_str.py similarity index 100% rename from Lib/test/dataclass_module_1_str.py rename to Lib/test/test_dataclasses/dataclass_module_1_str.py diff --git a/Lib/test/dataclass_module_2.py b/Lib/test/test_dataclasses/dataclass_module_2.py similarity index 100% rename from Lib/test/dataclass_module_2.py rename to Lib/test/test_dataclasses/dataclass_module_2.py diff --git a/Lib/test/dataclass_module_2_str.py b/Lib/test/test_dataclasses/dataclass_module_2_str.py similarity index 100% rename from Lib/test/dataclass_module_2_str.py rename to Lib/test/test_dataclasses/dataclass_module_2_str.py diff --git a/Lib/test/dataclass_textanno.py b/Lib/test/test_dataclasses/dataclass_textanno.py similarity index 100% rename from Lib/test/dataclass_textanno.py rename to Lib/test/test_dataclasses/dataclass_textanno.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 7b67738f4341a2..eb8f8c4fb943c0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2156,6 +2156,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_capi \ test/test_cppext \ test/test_ctypes \ + test/test_dataclasses \ test/test_email \ test/test_email/data \ test/test_import \ From b298b395e8ab1725c4f0dd736155b8c818664d42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 15:56:08 +0200 Subject: [PATCH 517/598] gh-108765: Cleanup #include in Python/*.c files (#108977) Mention one symbol imported by each #include. --- Include/internal/pycore_uops.h | 2 ++ Python/intrinsics.c | 8 ++++---- Python/legacy_tracing.c | 9 +++++---- Python/optimizer.c | 6 +++--- Python/pathconfig.c | 10 ++++++---- Python/perf_trampoline.c | 2 +- Python/pylifecycle.c | 5 +++-- Python/pystate.c | 6 +++--- Python/pystrtod.c | 3 ++- Python/pythonrun.c | 10 +++++----- Python/specialize.c | 6 +++--- Python/suggestions.c | 7 +++---- Python/symtable.c | 2 +- Python/sysmodule.c | 16 +++++++++------- Python/thread.c | 2 +- Python/thread_pthread_stubs.h | 2 +- Python/traceback.c | 4 ++-- Python/tracemalloc.c | 5 +++-- 18 files changed, 57 insertions(+), 48 deletions(-) diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index 254eeca2361bea..249f5c010e0092 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_frame.h" // _PyInterpreterFrame + #define _Py_UOP_MAX_TRACE_LENGTH 64 typedef struct { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 8e59c63aea39ff..bbd79ec473f470 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -5,11 +5,11 @@ #include "pycore_frame.h" #include "pycore_function.h" #include "pycore_global_objects.h" -#include "pycore_intrinsics.h" -#include "pycore_pyerrors.h" -#include "pycore_runtime.h" +#include "pycore_intrinsics.h" // INTRINSIC_PRINT +#include "pycore_pyerrors.h" // _PyErr_SetString() +#include "pycore_runtime.h" // _Py_ID() #include "pycore_sysmodule.h" // _PySys_GetAttr() -#include "pycore_typevarobject.h" +#include "pycore_typevarobject.h" // _Py_make_typevar() /******** Unary functions ********/ diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 17a13b1361f992..9258091afc7c49 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -2,12 +2,13 @@ * Provides callables to forward PEP 669 events to legacy events. */ -#include #include "Python.h" -#include "opcode.h" -#include "pycore_ceval.h" +#include "pycore_ceval.h" // export _PyEval_SetProfile() #include "pycore_object.h" -#include "pycore_sysmodule.h" +#include "pycore_sysmodule.h" // _PySys_Audit() + +#include "opcode.h" +#include typedef struct _PyLegacyEventHandler { PyObject_HEAD diff --git a/Python/optimizer.c b/Python/optimizer.c index 7472f52c50b766..8aaf9f9fd7cb14 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,9 +1,9 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode_metadata.h" -#include "pycore_opcode_utils.h" -#include "pycore_optimizer.h" +#include "pycore_opcode_metadata.h" // _PyOpcode_OpName() +#include "pycore_opcode_utils.h" // MAX_REAL_OPCODE +#include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uops.h" #include "cpython/optimizer.h" diff --git a/Python/pathconfig.c b/Python/pathconfig.c index afffa134e893ba..0ac64ec8110259 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -1,13 +1,15 @@ /* Path configuration like module_search_path (sys.path) */ #include "Python.h" -#include "marshal.h" // PyMarshal_ReadObjectFromString -#include "osdefs.h" // DELIM -#include "pycore_initconfig.h" -#include "pycore_fileutils.h" +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_fileutils.h" // _Py_wgetcwd() #include "pycore_pathconfig.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include + +#include "marshal.h" // PyMarshal_ReadObjectFromString +#include "osdefs.h" // DELIM + #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH # include diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 0f1af30226d9fb..209a23b6c1cbc7 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -130,7 +130,7 @@ any DWARF information available for them). */ #include "Python.h" -#include "pycore_ceval.h" +#include "pycore_ceval.h" // _PyPerf_Callbacks #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 64c74f433f222c..92eef6d50712bb 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -17,7 +17,7 @@ #include "pycore_list.h" // _PyList_Fini() #include "pycore_long.h" // _PyLong_InitTypes() #include "pycore_object.h" // _PyDebug_PrintTotalRefs() -#include "pycore_pathconfig.h" // _PyConfig_WritePathConfig() +#include "pycore_pathconfig.h" // _PyPathConfig_UpdateGlobal() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyObject_DebugMallocStats() @@ -32,13 +32,14 @@ #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_weakref.h" // _PyWeakref_GET_REF() + #include "opcode.h" #include // setlocale() #include // getenv() #if defined(__APPLE__) -#include +# include #endif #ifdef HAVE_SIGNAL_H diff --git a/Python/pystate.c b/Python/pystate.c index 46ea2c4f678db8..b2b9b9f8776c33 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -6,10 +6,10 @@ #include "pycore_code.h" // stats #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_frame.h" -#include "pycore_initconfig.h" +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() -#include "pycore_pyerrors.h" -#include "pycore_pylifecycle.h" +#include "pycore_pyerrors.h" // _PyErr_Clear() +#include "pycore_pylifecycle.h" // _PyAST_Fini() #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 9bb060e3d11979..16bf06f0e6cca2 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -3,7 +3,8 @@ #include #include "pycore_dtoa.h" // _Py_dg_strtod() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR -#include + +#include // localeconv() /* Case-insensitive string match used for nan and inf detection; t should be lower-case. Returns 1 for a successful match, 0 otherwise. */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 231ac78146a0e6..ddd8951cedb1f1 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -12,16 +12,16 @@ #include "Python.h" -#include "pycore_ast.h" // PyAST_mod2obj -#include "pycore_ceval.h" // _Py_EnterRecursiveCall +#include "pycore_ast.h" // PyAST_mod2obj() +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_object.h" // _PyDebug_PrintTotalRefs() #include "pycore_parser.h" // _PyParser_ASTFromString() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions -#include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pylifecycle.h" // _Py_FdIsInteractive() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_pythonrun.h" // define _PyRun_InteractiveLoopObject() +#include "pycore_pythonrun.h" // export _PyRun_InteractiveLoopObject() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() diff --git a/Python/specialize.c b/Python/specialize.c index a794f146c188ec..aa40b3343f3add 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -4,14 +4,14 @@ #include "pycore_code.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type -#include "pycore_dict.h" +#include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() -#include "pycore_global_strings.h" // _Py_ID() -#include "pycore_long.h" +#include "pycore_long.h" // _PyLong_IsNonNegativeCompact() #include "pycore_moduleobject.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() +#include "pycore_runtime.h" // _Py_ID() #include // rand() diff --git a/Python/suggestions.c b/Python/suggestions.c index 12097f793e3575..9247da4c7037a2 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -1,10 +1,9 @@ #include "Python.h" +#include "pycore_code.h" // _PyCode_GetVarnames() #include "pycore_frame.h" -#include "pycore_runtime.h" // _PyRuntime -#include "pycore_global_objects.h" // _Py_ID() +#include "pycore_pyerrors.h" // export _Py_UTF8_Edit_Cost() +#include "pycore_runtime.h" // _Py_ID() -#include "pycore_pyerrors.h" -#include "pycore_code.h" // _PyCode_GetVarnames() #include "stdlib_module_names.h" // _Py_stdlib_module_names #define MAX_CANDIDATE_ITEMS 750 diff --git a/Python/symtable.c b/Python/symtable.c index e9adbd5d29b1f9..f157d4c170314a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_ast.h" // identifier, stmt_ty +#include "pycore_ast.h" // stmt_ty #include "pycore_parser.h" // _PyParser_ASTFromString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntryObject diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0ec763c7aa7cf8..3835f760072ef9 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -23,7 +23,7 @@ Data members: #include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_object.h" // _PyObject_IS_GC(), _PyObject_DebugTypeStats() +#include "pycore_object.h" // _PyObject_DebugTypeStats() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _PyErr_WriteUnraisableDefaultHook() @@ -31,18 +31,19 @@ Data members: #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_structseq.h" // _PyStructSequence_InitBuiltinWithFlags() -#include "pycore_sysmodule.h" // Define _PySys_GetSizeOf() +#include "pycore_sysmodule.h" // export _PySys_GetSizeOf() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "frameobject.h" // PyFrame_FastToLocalsWithError() -#include "pydtrace.h" +#include "pydtrace.h" // PyDTrace_AUDIT() #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names + #include #ifdef MS_WINDOWS -#define WIN32_LEAN_AND_MEAN -#include +# define WIN32_LEAN_AND_MEAN +# include #endif /* MS_WINDOWS */ #ifdef MS_COREDLL @@ -52,11 +53,11 @@ extern const char *PyWin_DLLVersionString; #endif #ifdef __EMSCRIPTEN__ -#include +# include #endif #ifdef HAVE_FCNTL_H -#include +# include #endif /*[clinic input] @@ -66,6 +67,7 @@ module sys #include "clinic/sysmodule.c.h" + PyObject * _PySys_GetAttr(PyThreadState *tstate, PyObject *name) { diff --git a/Python/thread.c b/Python/thread.c index 7fc53f9b61360b..1ac2db2937e373 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -11,7 +11,7 @@ #include "pycore_pythread.h" #ifndef DONT_HAVE_STDIO_H -#include +# include #endif #include diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h index 56e5b6141924b4..48bad36ec449ab 100644 --- a/Python/thread_pthread_stubs.h +++ b/Python/thread_pthread_stubs.h @@ -40,7 +40,7 @@ pthread_cond_init(pthread_cond_t *restrict cond, return 0; } -PyAPI_FUNC(int)pthread_cond_destroy(pthread_cond_t *cond) +PyAPI_FUNC(int) pthread_cond_destroy(pthread_cond_t *cond) { return 0; } diff --git a/Python/traceback.c b/Python/traceback.c index 657ddab1cbf615..2fcfa7ca56c140 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -3,9 +3,9 @@ #include "Python.h" -#include "pycore_ast.h" // asdl_seq_* +#include "pycore_ast.h" // asdl_seq_GET() #include "pycore_call.h" // _PyObject_CallMethodFormat() -#include "pycore_compile.h" // _PyAST_Optimize +#include "pycore_compile.h" // _PyAST_Optimize() #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_frame.h" // _PyFrame_GetCode() #include "pycore_interp.h" // PyInterpreterState.gc diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 7d294ea5fe744c..19b64c619feb6a 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -2,11 +2,12 @@ #include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_object.h" // _PyType_PreHeaderSize +#include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() -#include "pycore_traceback.h" +#include "pycore_traceback.h" // _Py_DumpASCII() #include + #include "frameobject.h" // _PyInterpreterFrame_GetLine #include // malloc() From a8cae4071c795e55be46e339eda37e241fa0d7f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 15:57:01 +0200 Subject: [PATCH 518/598] gh-107219: Fix concurrent.futures terminate_broken() (#108974) Fix a race condition in _ExecutorManagerThread.terminate_broken(): ignore the InvalidStateError on future.set_exception(). It can happen if the future is cancelled before the caller. Moreover, test_crash_big_data() now waits explicitly until the executor completes. --- Lib/concurrent/futures/process.py | 9 ++++++++- Lib/test/test_concurrent_futures/test_deadlock.py | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 301207f59de37a..9933d3d0e040ea 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -489,7 +489,14 @@ def terminate_broken(self, cause): # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): - work_item.future.set_exception(bpe) + try: + work_item.future.set_exception(bpe) + except _base.InvalidStateError as exc: + # set_exception() fails if the future is cancelled: ignore it. + # Trying to check if the future is cancelled before calling + # set_exception() would leave a race condition if the future is + # cancelled betwen the check and set_exception(). + pass # Delete references to object. See issue16284 del work_item self.pending_work_items.clear() diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index 6b78b360d15627..baac2b51e0d4e2 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -239,6 +239,8 @@ def test_crash_big_data(self): with self.assertRaises(BrokenProcessPool): list(executor.map(_crash_with_data, [data] * 10)) + executor.shutdown(wait=True) + create_executor_tests(globals(), ExecutorDeadlockTest, executor_mixins=(ProcessPoolForkMixin, From fbce43a251488f666be9794c908a6613bf8ae260 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 16:34:35 +0200 Subject: [PATCH 519/598] gh-91960: Skip test_gdb if gdb cannot retrive Python frames (#108999) Skip test_gdb if gdb is unable to retrieve Python frame objects: if a frame is "". When Python is built with "clang -Og", gdb can fail to retrive the 'frame' parameter of _PyEval_EvalFrameDefault(). In this case, tests like py_bt() are likely to fail. Without getting access to Python frames, python-gdb.py is mostly clueless on retrieving the Python traceback. Moreover, test_gdb is no longer skipped on macOS if Python is built with Clang. --- Lib/test/test_gdb.py | 7 +++---- .../Tests/2023-09-06-15-36-51.gh-issue-91960.P3nD5v.rst | 7 +++++++ 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-06-15-36-51.gh-issue-91960.P3nD5v.rst diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index c05a2d387c429c..ca50574e46660d 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -55,10 +55,6 @@ def get_gdb_version(): if not sysconfig.is_python_build(): raise unittest.SkipTest("test_gdb only works on source builds at the moment.") -if 'Clang' in platform.python_compiler() and sys.platform == 'darwin': - raise unittest.SkipTest("test_gdb doesn't work correctly when python is" - " built with LLVM clang") - if ((sysconfig.get_config_var('PGO_PROF_USE_FLAG') or 'xxx') in (sysconfig.get_config_var('PY_CORE_CFLAGS') or '')): raise unittest.SkipTest("test_gdb is not reliable on PGO builds") @@ -247,6 +243,9 @@ def get_stack_trace(self, source=None, script=None, for pattern in ( '(frame information optimized out)', 'Unable to read information on python frame', + # gh-91960: On Python built with "clang -Og", gdb gets + # "frame=" for _PyEval_EvalFrameDefault() parameter + '(unable to read python frame information)', ): if pattern in out: raise unittest.SkipTest(f"{pattern!r} found in gdb output") diff --git a/Misc/NEWS.d/next/Tests/2023-09-06-15-36-51.gh-issue-91960.P3nD5v.rst b/Misc/NEWS.d/next/Tests/2023-09-06-15-36-51.gh-issue-91960.P3nD5v.rst new file mode 100644 index 00000000000000..46472abf9802bc --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-06-15-36-51.gh-issue-91960.P3nD5v.rst @@ -0,0 +1,7 @@ +Skip ``test_gdb`` if gdb is unable to retrieve Python frame objects: if a +frame is ````. When Python is built with "clang -Og", gdb can +fail to retrive the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In +this case, tests like ``py_bt()`` are likely to fail. Without getting access +to Python frames, ``python-gdb.py`` is mostly clueless on retrieving the +Python traceback. Moreover, ``test_gdb`` is no longer skipped on macOS if +Python is built with Clang. Patch by Victor Stinner. From f8a047941f2e4a1848700c21d58a08c9ec6a9c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 6 Sep 2023 16:49:44 +0200 Subject: [PATCH 520/598] gh-109002: Ensure only one wheel for each vendored package (#109003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Output with one wheel: ``` ❯ GITHUB_ACTIONS=true ./Tools/build/verify_ensurepip_wheels.py Verifying checksum for /Volumes/RAMDisk/cpython/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl. Expected digest: 7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be Actual digest: 7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be ::notice file=/Volumes/RAMDisk/cpython/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl::Successfully verified the checksum of the pip wheel. ``` Output with two wheels: ``` ❯ GITHUB_ACTIONS=true ./Tools/build/verify_ensurepip_wheels.py ::error file=/Volumes/RAMDisk/cpython/Lib/ensurepip/_bundled/pip-22.0.4-py3-none-any.whl::Found more than one wheel for package pip. ::error file=/Volumes/RAMDisk/cpython/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl::Found more than one wheel for package pip. ``` Output without wheels: ``` ❯ GITHUB_ACTIONS=true ./Tools/build/verify_ensurepip_wheels.py ::error file=::Could not find a pip wheel on disk. ``` --- Tools/build/verify_ensurepip_wheels.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index 09fd5d9e3103ac..29897425da6c03 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#!/usr/bin/env python3 """ Compare checksums for wheels in :mod:`ensurepip` against the Cheeseshop. @@ -35,11 +35,17 @@ def print_error(file_path: str, message: str) -> None: def verify_wheel(package_name: str) -> bool: # Find the package on disk - package_path = next(WHEEL_DIR.glob(f"{package_name}*.whl"), None) - if not package_path: - print_error("", f"Could not find a {package_name} wheel on disk.") + package_paths = list(WHEEL_DIR.glob(f"{package_name}*.whl")) + if len(package_paths) != 1: + if package_paths: + for p in package_paths: + print_error(p, f"Found more than one wheel for package {package_name}.") + else: + print_error("", f"Could not find a {package_name} wheel on disk.") return False + package_path = package_paths[0] + print(f"Verifying checksum for {package_path}.") # Find the version of the package used by ensurepip From 2cd170db40ffba357848672ff3d2f8c1e0e74f2c Mon Sep 17 00:00:00 2001 From: Ed Maste Date: Wed, 6 Sep 2023 10:57:40 -0400 Subject: [PATCH 521/598] gh-91960: Add FreeBSD build and test using Cirrus-CI (#91961) Cirrus-CI is a hosted CI service that supports FreeBSD, Linux, macOS, and Windows. Add a .cirrus.yml configuration file to provide CI coverage on pull requests for FreeBSD 13.2. Co-authored-by: Victor Stinner --- .cirrus.yml | 23 +++++++++++++++++++ ...3-09-05-21-42-54.gh-issue-91960.abClTs.rst | 1 + 2 files changed, 24 insertions(+) create mode 100644 .cirrus.yml create mode 100644 Misc/NEWS.d/next/Tests/2023-09-05-21-42-54.gh-issue-91960.abClTs.rst diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 00000000000000..823b1f921d66d1 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,23 @@ +freebsd_task: + freebsd_instance: + matrix: + - image: freebsd-13-2-release-amd64 + # Turn off TCP and UDP blackhole. It is not enabled by default in FreeBSD, + # but it is in the FreeBSD GCE images as used by Cirrus-CI. It causes even + # local local connections to fail with ETIMEDOUT instead of ECONNREFUSED. + # For more information see https://reviews.freebsd.org/D41751 and + # https://github.com/cirruslabs/cirrus-ci-docs/issues/483. + sysctl_script: + - sysctl net.inet.tcp.blackhole=0 + - sysctl net.inet.udp.blackhole=0 + build_script: + - mkdir build + - cd build + - ../configure --with-pydebug + - make -j$(sysctl -n hw.ncpu) + pythoninfo_script: + - cd build && make pythoninfo + test_script: + - cd build + # dtrace fails to build on FreeBSD - see gh-73263 + - make buildbottest TESTOPTS="-j0 -x test_dtrace --timeout=600" diff --git a/Misc/NEWS.d/next/Tests/2023-09-05-21-42-54.gh-issue-91960.abClTs.rst b/Misc/NEWS.d/next/Tests/2023-09-05-21-42-54.gh-issue-91960.abClTs.rst new file mode 100644 index 00000000000000..f63e0874499193 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-05-21-42-54.gh-issue-91960.abClTs.rst @@ -0,0 +1 @@ +FreeBSD 13.2 CI coverage for pull requests is now provided by Cirrus-CI (a hosted CI service that supports Linux, macOS, Windows, and FreeBSD). From 8ff11425783806f8cb78e99f667546b1f7f3428e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 17:34:31 +0200 Subject: [PATCH 522/598] gh-108851: Fix tomllib recursion tests (#108853) * Add get_recursion_available() and get_recursion_depth() functions to the test.support module. * Change infinite_recursion() default max_depth from 75 to 100. * Fix test_tomllib recursion tests for WASI buildbots: reduce the recursion limit and compute the maximum nested array/dict depending on the current available recursion limit. * test.pythoninfo logs sys.getrecursionlimit(). * Enhance test_sys tests on sys.getrecursionlimit() and sys.setrecursionlimit(). --- Lib/test/pythoninfo.py | 1 + Lib/test/support/__init__.py | 43 ++++++++++- Lib/test/test_support.py | 77 +++++++++++++++++++ Lib/test/test_sys.py | 65 ++++++++-------- Lib/test/test_tomllib/test_misc.py | 27 +++++-- ...-09-03-21-18-35.gh-issue-108851.CCuHyI.rst | 2 + ...-09-03-21-41-10.gh-issue-108851.xFTYOE.rst | 3 + 7 files changed, 177 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst create mode 100644 Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 53af21db0755b1..46522b50dd1e98 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -112,6 +112,7 @@ def collect_sys(info_add): call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') + call_func(info_add, 'sys.getrecursionlimit', sys, 'getrecursionlimit') encoding = sys.getfilesystemencoding() if hasattr(sys, 'getfilesystemencodeerrors'): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 7bac1160fd8e0a..d39f529499ca50 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2241,6 +2241,39 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) +def get_recursion_depth(): + """Get the recursion depth of the caller function. + + In the __main__ module, at the module level, it should be 1. + """ + try: + import _testinternalcapi + depth = _testinternalcapi.get_recursion_depth() + except (ImportError, RecursionError) as exc: + # sys._getframe() + frame.f_back implementation. + try: + depth = 0 + frame = sys._getframe() + while frame is not None: + depth += 1 + frame = frame.f_back + finally: + # Break any reference cycles. + frame = None + + # Ignore get_recursion_depth() frame. + return max(depth - 1, 1) + +def get_recursion_available(): + """Get the number of available frames before RecursionError. + + It depends on the current recursion depth of the caller function and + sys.getrecursionlimit(). + """ + limit = sys.getrecursionlimit() + depth = get_recursion_depth() + return limit - depth + @contextlib.contextmanager def set_recursion_limit(limit): """Temporarily change the recursion limit.""" @@ -2251,14 +2284,18 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=75): +def infinite_recursion(max_depth=100): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some debug windows builds, due to not enough functions being inlined the stack size might not handle the default recursion limit (1000). See bpo-11105 for details.""" - return set_recursion_limit(max_depth) - + if max_depth < 3: + raise ValueError("max_depth must be at least 3, got {max_depth}") + depth = get_recursion_depth() + depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. + limit = depth + max_depth + return set_recursion_limit(limit) def ignore_deprecations_from(module: str, *, like: str) -> object: token = object() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 86d26b7e8df4d0..64280739f00946 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -685,6 +685,83 @@ def test_has_strftime_extensions(self): else: self.assertTrue(support.has_strftime_extensions) + def test_get_recursion_depth(self): + # test support.get_recursion_depth() + code = textwrap.dedent(""" + from test import support + import sys + + def check(cond): + if not cond: + raise AssertionError("test failed") + + # depth 1 + check(support.get_recursion_depth() == 1) + + # depth 2 + def test_func(): + check(support.get_recursion_depth() == 2) + test_func() + + def test_recursive(depth, limit): + if depth >= limit: + # cannot call get_recursion_depth() at this depth, + # it can raise RecursionError + return + get_depth = support.get_recursion_depth() + print(f"test_recursive: {depth}/{limit}: " + f"get_recursion_depth() says {get_depth}") + check(get_depth == depth) + test_recursive(depth + 1, limit) + + # depth up to 25 + with support.infinite_recursion(max_depth=25): + limit = sys.getrecursionlimit() + print(f"test with sys.getrecursionlimit()={limit}") + test_recursive(2, limit) + + # depth up to 500 + with support.infinite_recursion(max_depth=500): + limit = sys.getrecursionlimit() + print(f"test with sys.getrecursionlimit()={limit}") + test_recursive(2, limit) + """) + script_helper.assert_python_ok("-c", code) + + def test_recursion(self): + # Test infinite_recursion() and get_recursion_available() functions. + def recursive_function(depth): + if depth: + recursive_function(depth - 1) + + for max_depth in (5, 25, 250): + with support.infinite_recursion(max_depth): + available = support.get_recursion_available() + + # Recursion up to 'available' additional frames should be OK. + recursive_function(available) + + # Recursion up to 'available+1' additional frames must raise + # RecursionError. Avoid self.assertRaises(RecursionError) which + # can consume more than 3 frames and so raises RecursionError. + try: + recursive_function(available + 1) + except RecursionError: + pass + else: + self.fail("RecursionError was not raised") + + # Test the bare minimumum: max_depth=3 + with support.infinite_recursion(3): + try: + recursive_function(3) + except RecursionError: + pass + else: + self.fail("RecursionError was not raised") + + #self.assertEqual(available, 2) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index e8a99244a3a28d..e4a341bd3e3db8 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -279,20 +279,29 @@ def test_switchinterval(self): finally: sys.setswitchinterval(orig) - def test_recursionlimit(self): + def test_getrecursionlimit(self): + limit = sys.getrecursionlimit() + self.assertIsInstance(limit, int) + self.assertGreater(limit, 1) + self.assertRaises(TypeError, sys.getrecursionlimit, 42) - oldlimit = sys.getrecursionlimit() - self.assertRaises(TypeError, sys.setrecursionlimit) - self.assertRaises(ValueError, sys.setrecursionlimit, -42) - sys.setrecursionlimit(10000) - self.assertEqual(sys.getrecursionlimit(), 10000) - sys.setrecursionlimit(oldlimit) + + def test_setrecursionlimit(self): + old_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(10_005) + self.assertEqual(sys.getrecursionlimit(), 10_005) + + self.assertRaises(TypeError, sys.setrecursionlimit) + self.assertRaises(ValueError, sys.setrecursionlimit, -42) + finally: + sys.setrecursionlimit(old_limit) def test_recursionlimit_recovery(self): if hasattr(sys, 'gettrace') and sys.gettrace(): self.skipTest('fatal error if run with a trace function') - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() def f(): f() try: @@ -311,35 +320,31 @@ def f(): with self.assertRaises(RecursionError): f() finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) @test.support.cpython_only - def test_setrecursionlimit_recursion_depth(self): + def test_setrecursionlimit_to_depth(self): # Issue #25274: Setting a low recursion limit must be blocked if the # current recursion depth is already higher than limit. - from _testinternalcapi import get_recursion_depth - - def set_recursion_limit_at_depth(depth, limit): - recursion_depth = get_recursion_depth() - if recursion_depth >= depth: - with self.assertRaises(RecursionError) as cm: - sys.setrecursionlimit(limit) - self.assertRegex(str(cm.exception), - "cannot set the recursion limit to [0-9]+ " - "at the recursion depth [0-9]+: " - "the limit is too low") - else: - set_recursion_limit_at_depth(depth, limit) - - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() try: - sys.setrecursionlimit(1000) - - for limit in (10, 25, 50, 75, 100, 150, 200): - set_recursion_limit_at_depth(limit, limit) + depth = support.get_recursion_depth() + with self.subTest(limit=sys.getrecursionlimit(), depth=depth): + # depth + 1 is OK + sys.setrecursionlimit(depth + 1) + + # reset the limit to be able to call self.assertRaises() + # context manager + sys.setrecursionlimit(old_limit) + with self.assertRaises(RecursionError) as cm: + sys.setrecursionlimit(depth) + self.assertRegex(str(cm.exception), + "cannot set the recursion limit to [0-9]+ " + "at the recursion depth [0-9]+: " + "the limit is too low") finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) def test_getwindowsversion(self): # Raise SkipTest if sys doesn't have getwindowsversion attribute diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index a477a219fd9ebd..9e677a337a2835 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -9,6 +9,7 @@ import sys import tempfile import unittest +from test import support from . import tomllib @@ -92,13 +93,23 @@ def test_deepcopy(self): self.assertEqual(obj_copy, expected_obj) def test_inline_array_recursion_limit(self): - # 465 with default recursion limit - nest_count = int(sys.getrecursionlimit() * 0.465) - recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" - tomllib.loads(recursive_array_toml) + with support.infinite_recursion(max_depth=100): + available = support.get_recursion_available() + nest_count = (available // 2) - 2 + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=available, + nest_count=nest_count): + recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" + tomllib.loads(recursive_array_toml) def test_inline_table_recursion_limit(self): - # 310 with default recursion limit - nest_count = int(sys.getrecursionlimit() * 0.31) - recursive_table_toml = nest_count * "key = {" + nest_count * "}" - tomllib.loads(recursive_table_toml) + with support.infinite_recursion(max_depth=100): + available = support.get_recursion_available() + nest_count = (available // 3) - 1 + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=available, + nest_count=nest_count): + recursive_table_toml = nest_count * "key = {" + nest_count * "}" + tomllib.loads(recursive_table_toml) diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst b/Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst new file mode 100644 index 00000000000000..7a5b3052af22f2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst @@ -0,0 +1,2 @@ +Add ``get_recursion_available()`` and ``get_recursion_depth()`` functions to +the :mod:`test.support` module. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst b/Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst new file mode 100644 index 00000000000000..b35aaebb410afb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst @@ -0,0 +1,3 @@ +Fix ``test_tomllib`` recursion tests for WASI buildbots: reduce the recursion +limit and compute the maximum nested array/dict depending on the current +available recursion limit. Patch by Victor Stinner. From a0773b89dfe5cd2190d539905dd89e7f6455668e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 17:54:59 +0200 Subject: [PATCH 523/598] gh-108753: Enhance pystats (#108754) Statistics gathering is now off by default. Use the "-X pystats" command line option or set the new PYTHONSTATS environment variable to 1 to turn statistics gathering on at Python startup. Statistics are no longer dumped at exit if statistics gathering was off or statistics have been cleared. Changes: * Add PYTHONSTATS environment variable. * sys._stats_dump() now returns False if statistics are not dumped because they are all equal to zero. * Add PyConfig._pystats member. * Add tests on sys functions and on setting PyConfig._pystats to 1. * Add Include/cpython/pystats.h and Include/internal/pycore_pystats.h header files. * Rename '_py_stats' variable to '_Py_stats'. * Exclude Include/cpython/pystats.h from the Py_LIMITED_API. * Move pystats.h include from object.h to Python.h. * Add _Py_StatsOn() and _Py_StatsOff() functions. Remove '_py_stats_struct' variable from the API: make it static in specialize.c. * Document API in Include/pystats.h and Include/cpython/pystats.h. * Complete pystats documentation in Doc/using/configure.rst. * Don't write "all zeros" stats: if _stats_off() and _stats_clear() or _stats_dump() were called. * _PyEval_Fini() now always call _Py_PrintSpecializationStats() which does nothing if stats are all zeros. Co-authored-by: Michael Droettboom --- Doc/using/configure.rst | 61 +++++++++++++- Include/cpython/initconfig.h | 5 ++ Include/cpython/pystats.h | 120 ++++++++++++++++++++++++++++ Include/internal/pycore_code.h | 18 ++--- Include/internal/pycore_pystats.h | 21 +++++ Include/pystats.h | 124 +++-------------------------- Lib/test/test_embed.py | 9 +++ Lib/test/test_sys.py | 9 +++ Makefile.pre.in | 2 + Modules/gcmodule.c | 10 +-- PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 6 ++ Programs/_testembed.c | 7 ++ Python/ceval_gil.c | 5 +- Python/ceval_macros.h | 2 +- Python/clinic/sysmodule.c.h | 24 ++++-- Python/initconfig.c | 34 +++++++- Python/specialize.c | 98 +++++++++++++++-------- Python/sysmodule.c | 28 ++++--- 19 files changed, 402 insertions(+), 183 deletions(-) create mode 100644 Include/cpython/pystats.h create mode 100644 Include/internal/pycore_pystats.h diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index fe35372603fdd8..ad58255c33b33b 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -192,14 +192,69 @@ General Options .. cmdoption:: --enable-pystats - Turn on internal statistics gathering. + Turn on internal Python performance statistics gathering. + + By default, statistics gathering is off. Use ``python3 -X pystats`` command + or set ``PYTHONSTATS=1`` environment variable to turn on statistics + gathering at Python startup. + + At Python exit, dump statistics if statistics gathering was on and not + cleared. + + Effects: + + * Add :option:`-X pystats <-X>` command line option. + * Add :envvar:`!PYTHONSTATS` environment variable. + * Define the ``Py_STATS`` macro. + * Add functions to the :mod:`sys` module: + + * :func:`!sys._stats_on`: Turns on statistics gathering. + * :func:`!sys._stats_off`: Turns off statistics gathering. + * :func:`!sys._stats_clear`: Clears the statistics. + * :func:`!sys._stats_dump`: Dump statistics to file, and clears the statistics. The statistics will be dumped to a arbitrary (probably unique) file in - ``/tmp/py_stats/``, or ``C:\temp\py_stats\`` on Windows. If that directory - does not exist, results will be printed on stdout. + ``/tmp/py_stats/`` (Unix) or ``C:\temp\py_stats\`` (Windows). If that + directory does not exist, results will be printed on stderr. Use ``Tools/scripts/summarize_stats.py`` to read the stats. + Statistics: + + * Opcode: + + * Specialization: success, failure, hit, deferred, miss, deopt, failures; + * Execution count; + * Pair count. + + * Call: + + * Inlined Python calls; + * PyEval calls; + * Frames pushed; + * Frame object created; + * Eval calls: vector, generator, legacy, function VECTORCALL, build class, + slot, function "ex", API, method. + + * Object: + + * incref and decref; + * interpreter incref and decref; + * allocations: all, 512 bytes, 4 kiB, big; + * free; + * to/from free lists; + * dictionary materialized/dematerialized; + * type cache; + * optimization attemps; + * optimization traces created/executed; + * uops executed. + + * Garbage collector: + + * Garbage collections; + * Objects visited; + * Objects collected. + .. versionadded:: 3.11 .. cmdoption:: --disable-gil diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 7fb7a9868be926..ee130467824daa 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -215,6 +215,11 @@ typedef struct PyConfig { // If non-zero, we believe we're running from a source tree. int _is_python_build; + +#ifdef Py_STATS + // If non-zero, turns on statistics gathering. + int _pystats; +#endif } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h new file mode 100644 index 00000000000000..150e16faa96ca1 --- /dev/null +++ b/Include/cpython/pystats.h @@ -0,0 +1,120 @@ +// Statistics on Python performance. +// +// API: +// +// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() +// and Py_DECREF(). +// - _Py_stats variable +// +// Functions of the sys module: +// +// - sys._stats_on() +// - sys._stats_off() +// - sys._stats_clear() +// - sys._stats_dump() +// +// Python must be built with ./configure --enable-pystats to define the +// Py_STATS macro. +// +// Define _PY_INTERPRETER macro to increment interpreter_increfs and +// interpreter_decrefs. Otherwise, increment increfs and decrefs. + +#ifndef Py_CPYTHON_PYSTATS_H +# error "this header file must not be included directly" +#endif + +#define SPECIALIZATION_FAILURE_KINDS 36 + +/* Stats for determining who is calling PyEval_EvalFrame */ +#define EVAL_CALL_TOTAL 0 +#define EVAL_CALL_VECTOR 1 +#define EVAL_CALL_GENERATOR 2 +#define EVAL_CALL_LEGACY 3 +#define EVAL_CALL_FUNCTION_VECTORCALL 4 +#define EVAL_CALL_BUILD_CLASS 5 +#define EVAL_CALL_SLOT 6 +#define EVAL_CALL_FUNCTION_EX 7 +#define EVAL_CALL_API 8 +#define EVAL_CALL_METHOD 9 + +#define EVAL_CALL_KINDS 10 + +typedef struct _specialization_stats { + uint64_t success; + uint64_t failure; + uint64_t hit; + uint64_t deferred; + uint64_t miss; + uint64_t deopt; + uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS]; +} SpecializationStats; + +typedef struct _opcode_stats { + SpecializationStats specialization; + uint64_t execution_count; + uint64_t pair_count[256]; +} OpcodeStats; + +typedef struct _call_stats { + uint64_t inlined_py_calls; + uint64_t pyeval_calls; + uint64_t frames_pushed; + uint64_t frame_objects_created; + uint64_t eval_calls[EVAL_CALL_KINDS]; +} CallStats; + +typedef struct _object_stats { + uint64_t increfs; + uint64_t decrefs; + uint64_t interpreter_increfs; + uint64_t interpreter_decrefs; + uint64_t allocations; + uint64_t allocations512; + uint64_t allocations4k; + uint64_t allocations_big; + uint64_t frees; + uint64_t to_freelist; + uint64_t from_freelist; + uint64_t new_values; + uint64_t dict_materialized_on_request; + uint64_t dict_materialized_new_key; + uint64_t dict_materialized_too_big; + uint64_t dict_materialized_str_subclass; + uint64_t dict_dematerialized; + uint64_t type_cache_hits; + uint64_t type_cache_misses; + uint64_t type_cache_dunder_hits; + uint64_t type_cache_dunder_misses; + uint64_t type_cache_collisions; + uint64_t optimization_attempts; + uint64_t optimization_traces_created; + uint64_t optimization_traces_executed; + uint64_t optimization_uops_executed; + /* Temporary value used during GC */ + uint64_t object_visits; +} ObjectStats; + +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + +typedef struct _stats { + OpcodeStats opcode_stats[256]; + CallStats call_stats; + ObjectStats object_stats; + GCStats *gc_stats; +} PyStats; + + +// Export for shared extensions like 'math' +PyAPI_DATA(PyStats*) _Py_stats; + +#ifdef _PY_INTERPRETER +# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) +# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) +#else +# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) +# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) +#endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index f5127a81144353..7c6629074758da 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -268,17 +268,17 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #ifdef Py_STATS -#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0) +#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) +#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) +#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) +#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) +#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) #define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0) + do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) +#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0) -#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0) + do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) +#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h new file mode 100644 index 00000000000000..f8af398a560586 --- /dev/null +++ b/Include/internal/pycore_pystats.h @@ -0,0 +1,21 @@ +#ifndef Py_INTERNAL_PYSTATS_H +#define Py_INTERNAL_PYSTATS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_STATS +extern void _Py_StatsOn(void); +extern void _Py_StatsOff(void); +extern void _Py_StatsClear(void); +extern int _Py_PrintSpecializationStats(int to_file); +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_PYSTATS_H diff --git a/Include/pystats.h b/Include/pystats.h index b1957596745f00..acfa32201711e0 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -1,4 +1,9 @@ - +// Statistics on Python performance (public API). +// +// Define _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() +// and Py_DECREF(). +// +// See Include/cpython/pystats.h for the full API. #ifndef Py_PYSTATS_H #define Py_PYSTATS_H @@ -6,119 +11,16 @@ extern "C" { #endif -#ifdef Py_STATS - -#define SPECIALIZATION_FAILURE_KINDS 36 - -/* Stats for determining who is calling PyEval_EvalFrame */ -#define EVAL_CALL_TOTAL 0 -#define EVAL_CALL_VECTOR 1 -#define EVAL_CALL_GENERATOR 2 -#define EVAL_CALL_LEGACY 3 -#define EVAL_CALL_FUNCTION_VECTORCALL 4 -#define EVAL_CALL_BUILD_CLASS 5 -#define EVAL_CALL_SLOT 6 -#define EVAL_CALL_FUNCTION_EX 7 -#define EVAL_CALL_API 8 -#define EVAL_CALL_METHOD 9 - -#define EVAL_CALL_KINDS 10 - -typedef struct _specialization_stats { - uint64_t success; - uint64_t failure; - uint64_t hit; - uint64_t deferred; - uint64_t miss; - uint64_t deopt; - uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS]; -} SpecializationStats; - -typedef struct _opcode_stats { - SpecializationStats specialization; - uint64_t execution_count; - uint64_t pair_count[256]; -} OpcodeStats; - -typedef struct _call_stats { - uint64_t inlined_py_calls; - uint64_t pyeval_calls; - uint64_t frames_pushed; - uint64_t frame_objects_created; - uint64_t eval_calls[EVAL_CALL_KINDS]; -} CallStats; - -typedef struct _object_stats { - uint64_t increfs; - uint64_t decrefs; - uint64_t interpreter_increfs; - uint64_t interpreter_decrefs; - uint64_t allocations; - uint64_t allocations512; - uint64_t allocations4k; - uint64_t allocations_big; - uint64_t frees; - uint64_t to_freelist; - uint64_t from_freelist; - uint64_t new_values; - uint64_t dict_materialized_on_request; - uint64_t dict_materialized_new_key; - uint64_t dict_materialized_too_big; - uint64_t dict_materialized_str_subclass; - uint64_t dict_dematerialized; - uint64_t type_cache_hits; - uint64_t type_cache_misses; - uint64_t type_cache_dunder_hits; - uint64_t type_cache_dunder_misses; - uint64_t type_cache_collisions; - uint64_t optimization_attempts; - uint64_t optimization_traces_created; - uint64_t optimization_traces_executed; - uint64_t optimization_uops_executed; - /* Temporary value used during GC */ - uint64_t object_visits; -} ObjectStats; - -typedef struct _gc_stats { - uint64_t collections; - uint64_t object_visits; - uint64_t objects_collected; -} GCStats; - -typedef struct _stats { - OpcodeStats opcode_stats[256]; - CallStats call_stats; - ObjectStats object_stats; - GCStats *gc_stats; -} PyStats; - - -PyAPI_DATA(PyStats) _py_stats_struct; -PyAPI_DATA(PyStats *) _py_stats; - -extern void _Py_StatsClear(void); -extern void _Py_PrintSpecializationStats(int to_file); - -#ifdef _PY_INTERPRETER - -#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0) -#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0) - +#if defined(Py_STATS) && !defined(Py_LIMITED_API) +# define Py_CPYTHON_PYSTATS_H +# include "cpython/pystats.h" +# undef Py_CPYTHON_PYSTATS_H #else - -#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0) -#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0) - -#endif - -#else - -#define _Py_INCREF_STAT_INC() ((void)0) -#define _Py_DECREF_STAT_INC() ((void)0) - +# define _Py_INCREF_STAT_INC() ((void)0) +# define _Py_DECREF_STAT_INC() ((void)0) #endif // !Py_STATS #ifdef __cplusplus } #endif -#endif /* !Py_PYSTATs_H */ +#endif // !Py_PYSTATS_H diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 50c9f61017e022..7f1a4e665f3b5d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -26,6 +26,7 @@ PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 PYMEM_ALLOCATOR_MALLOC = 3 +Py_STATS = hasattr(sys, '_stats_on') # _PyCoreConfig_InitCompatConfig() API_COMPAT = 1 @@ -512,6 +513,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'safe_path': 0, '_is_python_build': IGNORE_CONFIG, } + if Py_STATS: + CONFIG_COMPAT['_pystats'] = 0 if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, @@ -895,6 +898,8 @@ def test_init_from_config(self): 'check_hash_pycs_mode': 'always', 'pathconfig_warnings': 0, } + if Py_STATS: + config['_pystats'] = 1 self.check_all_configs("test_init_from_config", config, preconfig, api=API_COMPAT) @@ -927,6 +932,8 @@ def test_init_compat_env(self): 'safe_path': 1, 'int_max_str_digits': 4567, } + if Py_STATS: + config['_pystats'] = 1 self.check_all_configs("test_init_compat_env", config, preconfig, api=API_COMPAT) @@ -960,6 +967,8 @@ def test_init_python_env(self): 'safe_path': 1, 'int_max_str_digits': 4567, } + if Py_STATS: + config['_pystats'] = 1 self.check_all_configs("test_init_python_env", config, preconfig, api=API_PYTHON) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index e4a341bd3e3db8..f4948ceec66226 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1200,6 +1200,15 @@ class MyType: get_objects = sys.getobjects(3, MyType) self.assertEqual(len(get_objects), 3) + @unittest.skipUnless(hasattr(sys, '_stats_on'), 'need Py_STATS build') + def test_pystats(self): + # Call the functions, just check that they don't crash + # Cannot save/restore state. + sys._stats_on() + sys._stats_off() + sys._stats_clear() + sys._stats_dump() + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index eb8f8c4fb943c0..cf77cbd13ad4d9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1724,6 +1724,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pylifecycle.h \ $(srcdir)/Include/cpython/pymem.h \ $(srcdir)/Include/cpython/pystate.h \ + $(srcdir)/Include/cpython/pystats.h \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ @@ -1798,6 +1799,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pymem.h \ $(srcdir)/Include/internal/pycore_pymem_init.h \ $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pystats.h \ $(srcdir)/Include/internal/pycore_pythonrun.h \ $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_range.h \ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 35a35091bf4511..632cabdf4bcfbd 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1200,8 +1200,8 @@ gc_collect_main(PyThreadState *tstate, int generation, { GC_STAT_ADD(generation, collections, 1); #ifdef Py_STATS - if (_py_stats) { - _py_stats->object_stats.object_visits = 0; + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; } #endif int i; @@ -1362,10 +1362,10 @@ gc_collect_main(PyThreadState *tstate, int generation, GC_STAT_ADD(generation, objects_collected, m); #ifdef Py_STATS - if (_py_stats) { + if (_Py_stats) { GC_STAT_ADD(generation, object_visits, - _py_stats->object_stats.object_visits); - _py_stats->object_stats.object_visits = 0; + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; } #endif diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 4cd095b28e510f..04752a8029acc2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -176,6 +176,7 @@ + @@ -261,6 +262,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index af1669209a9049..4ad02778466925 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -492,6 +492,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -693,6 +696,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7ee64b22925f0c..bc991020d0fa77 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -708,6 +708,10 @@ static int test_init_from_config(void) config.pathconfig_warnings = 0; config.safe_path = 1; +#ifdef Py_STATS + putenv("PYTHONSTATS="); + config._pystats = 1; +#endif putenv("PYTHONINTMAXSTRDIGITS=6666"); config.int_max_str_digits = 31337; @@ -778,6 +782,9 @@ static void set_most_env_vars(void) putenv("PYTHONPLATLIBDIR=env_platlibdir"); putenv("PYTHONSAFEPATH=1"); putenv("PYTHONINTMAXSTRDIGITS=4567"); +#ifdef Py_STATS + putenv("PYTHONSTATS=1"); +#endif } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 7c9ad07cc7207b..e53ffa76b1164b 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -2,11 +2,12 @@ #include "Python.h" #include "pycore_atomic.h" // _Py_atomic_int #include "pycore_ceval.h" // _PyEval_SignalReceived() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() +#include "pycore_pystats.h" // _Py_PrintSpecializationStats() /* Notes about the implementation: diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 4b7c4448e0ea25..81fbb7982ad11c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -64,7 +64,7 @@ do { \ frame->prev_instr = next_instr++; \ OPCODE_EXE_INC(op); \ - if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \ + if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a4b39873735fd9..30691c3d08ae67 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1120,7 +1120,7 @@ PyDoc_STRVAR(sys__stats_on__doc__, "_stats_on($module, /)\n" "--\n" "\n" -"Turns on stats gathering (stats gathering is on by default)."); +"Turns on stats gathering (stats gathering is off by default)."); #define SYS__STATS_ON_METHODDEF \ {"_stats_on", (PyCFunction)sys__stats_on, METH_NOARGS, sys__stats_on__doc__}, @@ -1142,7 +1142,7 @@ PyDoc_STRVAR(sys__stats_off__doc__, "_stats_off($module, /)\n" "--\n" "\n" -"Turns off stats gathering (stats gathering is on by default)."); +"Turns off stats gathering (stats gathering is off by default)."); #define SYS__STATS_OFF_METHODDEF \ {"_stats_off", (PyCFunction)sys__stats_off, METH_NOARGS, sys__stats_off__doc__}, @@ -1186,18 +1186,30 @@ PyDoc_STRVAR(sys__stats_dump__doc__, "_stats_dump($module, /)\n" "--\n" "\n" -"Dump stats to file, and clears the stats."); +"Dump stats to file, and clears the stats.\n" +"\n" +"Return False if no statistics were not dumped because stats gathering was off."); #define SYS__STATS_DUMP_METHODDEF \ {"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__}, -static PyObject * +static int sys__stats_dump_impl(PyObject *module); static PyObject * sys__stats_dump(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return sys__stats_dump_impl(module); + PyObject *return_value = NULL; + int _return_value; + + _return_value = sys__stats_dump_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; } #endif /* defined(Py_STATS) */ @@ -1411,4 +1423,4 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=6619682ea70e7375 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=549bb1f92a15f916 input=a9049054013a1b77]*/ diff --git a/Python/initconfig.c b/Python/initconfig.c index f9c5c64f9c8759..a0467f51d4834e 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -9,6 +9,7 @@ #include "pycore_pylifecycle.h" // _Py_PreInitializeFromConfig() #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pystats.h" // _Py_StatsOn() #include "osdefs.h" // DELIM @@ -186,7 +187,11 @@ static const char usage_envvars[] = "PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" "PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNINGS=arg : warning control (-W arg)\n"; +"PYTHONWARNINGS=arg : warning control (-W arg)\n" +#ifdef Py_STATS +"PYTHONSTATS : turns on statistics gathering\n" +#endif +; #if defined(MS_WINDOWS) # define PYTHONHOMEHELP "\\python{major}{minor}" @@ -630,6 +635,9 @@ config_check_consistency(const PyConfig *config) assert(config->int_max_str_digits >= 0); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). +#ifdef Py_STATS + assert(config->_pystats >= 0); +#endif return 1; } #endif @@ -951,6 +959,9 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTRLIST(orig_argv); COPY_ATTR(_is_python_build); COPY_ATTR(int_max_str_digits); +#ifdef Py_STATS + COPY_ATTR(_pystats); +#endif #undef COPY_ATTR #undef COPY_WSTR_ATTR @@ -1058,6 +1069,9 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(safe_path); SET_ITEM_INT(_is_python_build); SET_ITEM_INT(int_max_str_digits); +#ifdef Py_STATS + SET_ITEM_INT(_pystats); +#endif return dict; @@ -1365,6 +1379,9 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(safe_path); GET_UINT(_is_python_build); GET_INT(int_max_str_digits); +#ifdef Py_STATS + GET_UINT(_pystats); +#endif #undef CHECK_VALUE #undef GET_UINT @@ -2116,7 +2133,13 @@ config_read(PyConfig *config, int compute_path_config) #ifdef Py_STATS if (config_get_xoption(config, L"pystats")) { - _py_stats = &_py_stats_struct; + config->_pystats = 1; + } + else if (config_get_env(config, "PYTHONSTATS")) { + config->_pystats = 1; + } + if (config->_pystats < 0) { + config->_pystats = 0; } #endif @@ -2254,6 +2277,13 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) { return _PyStatus_NO_MEMORY(); } + +#ifdef Py_STATS + if (config->_pystats) { + _Py_StatsOn(); + } +#endif + return _PyStatus_OK(); } diff --git a/Python/specialize.c b/Python/specialize.c index aa40b3343f3add..e072167ff38ce9 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -13,17 +13,17 @@ #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() - #include // rand() + /* For guidance on adding or extending families of instructions see * ./adaptive.md */ #ifdef Py_STATS GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; -PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] }; -PyStats *_py_stats = NULL; +static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; +PyStats *_Py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ do { \ @@ -83,7 +83,7 @@ add_stat_dict( int opcode, const char *name) { - SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization; + SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization; PyObject *d = stats_to_dict(stats); if (d == NULL) { return -1; @@ -93,7 +93,6 @@ add_stat_dict( return err; } -#ifdef Py_STATS PyObject* _Py_GetSpecializationStats(void) { PyObject *stats = PyDict_New(); @@ -120,7 +119,6 @@ _Py_GetSpecializationStats(void) { } return stats; } -#endif #define PRINT_STAT(i, field) \ @@ -218,26 +216,65 @@ print_gc_stats(FILE *out, GCStats *stats) } static void -print_stats(FILE *out, PyStats *stats) { +print_stats(FILE *out, PyStats *stats) +{ print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); } +void +_Py_StatsOn(void) +{ + _Py_stats = &_Py_stats_struct; +} + +void +_Py_StatsOff(void) +{ + _Py_stats = NULL; +} + void _Py_StatsClear(void) { - for (int i = 0; i < NUM_GENERATIONS; i++) { - _py_gc_stats[i] = (GCStats) { 0 }; + memset(&_py_gc_stats, 0, sizeof(_py_gc_stats)); + memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct)); + _Py_stats_struct.gc_stats = _py_gc_stats; +} + +static int +mem_is_zero(unsigned char *ptr, size_t size) +{ + for (size_t i=0; i < size; i++) { + if (*ptr != 0) { + return 0; + } + ptr++; } - _py_stats_struct = (PyStats) { 0 }; - _py_stats_struct.gc_stats = _py_gc_stats; + return 1; } -void +int _Py_PrintSpecializationStats(int to_file) { + PyStats *stats = &_Py_stats_struct; +#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) + int is_zero = ( + MEM_IS_ZERO(stats->gc_stats) // is a pointer + && MEM_IS_ZERO(&stats->opcode_stats) + && MEM_IS_ZERO(&stats->call_stats) + && MEM_IS_ZERO(&stats->object_stats) + ); +#undef MEM_IS_ZERO + if (is_zero) { + // gh-108753: -X pystats command line was used, but then _stats_off() + // and _stats_clear() have been called: in this case, avoid printing + // useless "all zeros" statistics. + return 0; + } + FILE *out = stderr; if (to_file) { /* Write to a file instead of stderr. */ @@ -268,26 +305,25 @@ _Py_PrintSpecializationStats(int to_file) else { fprintf(out, "Specialization stats:\n"); } - print_stats(out, &_py_stats_struct); + print_stats(out, stats); if (out != stderr) { fclose(out); } + return 1; } -#ifdef Py_STATS - #define SPECIALIZATION_FAIL(opcode, kind) \ do { \ - if (_py_stats) { \ - _py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \ + if (_Py_stats) { \ + _Py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \ } \ } while (0) -#endif -#endif +#endif // Py_STATS + #ifndef SPECIALIZATION_FAIL -#define SPECIALIZATION_FAIL(opcode, kind) ((void)0) +# define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif // Initialize warmup counters and insert superinstructions. This cannot fail. @@ -1067,7 +1103,7 @@ load_attr_fail_kind(DescriptorClassification kind) } Py_UNREACHABLE(); } -#endif +#endif // Py_STATS static int specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, @@ -1306,7 +1342,7 @@ binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub) } return SPEC_FAIL_OTHER; } -#endif +#endif // Py_STATS static int function_kind(PyCodeObject *code) { @@ -1545,7 +1581,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } goto fail; } -#endif +#endif // Py_STATS SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); fail: STAT_INC(STORE_SUBSCR, failure); @@ -1690,7 +1726,7 @@ meth_descr_call_fail_kind(int ml_flags) return SPEC_FAIL_CALL_BAD_CALL_FLAGS; } } -#endif +#endif // Py_STATS static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, @@ -1871,7 +1907,7 @@ call_fail_kind(PyObject *callable) } return SPEC_FAIL_OTHER; } -#endif +#endif // Py_STATS /* TODO: @@ -1995,7 +2031,7 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) } Py_UNREACHABLE(); } -#endif +#endif // Py_STATS void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, @@ -2102,7 +2138,7 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) } return SPEC_FAIL_OTHER; } -#endif +#endif // Py_STATS void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, @@ -2165,7 +2201,7 @@ unpack_sequence_fail_kind(PyObject *seq) } return SPEC_FAIL_OTHER; } -#endif +#endif // Py_STATS void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) @@ -2206,7 +2242,6 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) } #ifdef Py_STATS - int _PySpecialization_ClassifyIterator(PyObject *iter) { @@ -2277,8 +2312,7 @@ int } return SPEC_FAIL_OTHER; } - -#endif +#endif // Py_STATS void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) @@ -2431,7 +2465,7 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) goto failure; } SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OTHER); -#endif +#endif // Py_STATS failure: STAT_INC(TO_BOOL, failure); instr->op.code = TO_BOOL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 3835f760072ef9..fed12812a77089 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -30,6 +30,7 @@ Data members: #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pystats.h" // _Py_PrintSpecializationStats() #include "pycore_structseq.h" // _PyStructSequence_InitBuiltinWithFlags() #include "pycore_sysmodule.h" // export _PySys_GetSizeOf() #include "pycore_tuple.h" // _PyTuple_FromArray() @@ -2106,32 +2107,33 @@ sys_is_finalizing_impl(PyObject *module) return PyBool_FromLong(Py_IsFinalizing()); } + #ifdef Py_STATS /*[clinic input] sys._stats_on -Turns on stats gathering (stats gathering is on by default). +Turns on stats gathering (stats gathering is off by default). [clinic start generated code]*/ static PyObject * sys__stats_on_impl(PyObject *module) -/*[clinic end generated code: output=aca53eafcbb4d9fe input=8ddc6df94e484f3a]*/ +/*[clinic end generated code: output=aca53eafcbb4d9fe input=43b5bfe145299e55]*/ { - _py_stats = &_py_stats_struct; + _Py_StatsOn(); Py_RETURN_NONE; } /*[clinic input] sys._stats_off -Turns off stats gathering (stats gathering is on by default). +Turns off stats gathering (stats gathering is off by default). [clinic start generated code]*/ static PyObject * sys__stats_off_impl(PyObject *module) -/*[clinic end generated code: output=1534c1ee63812214 input=b3e50e71ecf29f66]*/ +/*[clinic end generated code: output=1534c1ee63812214 input=d1a84c60c56cbce2]*/ { - _py_stats = NULL; + _Py_StatsOff(); Py_RETURN_NONE; } @@ -2150,21 +2152,23 @@ sys__stats_clear_impl(PyObject *module) } /*[clinic input] -sys._stats_dump +sys._stats_dump -> bool Dump stats to file, and clears the stats. + +Return False if no statistics were not dumped because stats gathering was off. [clinic start generated code]*/ -static PyObject * +static int sys__stats_dump_impl(PyObject *module) -/*[clinic end generated code: output=79f796fb2b4ddf05 input=92346f16d64f6f95]*/ +/*[clinic end generated code: output=6e346b4ba0de4489 input=31a489e39418b2a5]*/ { - _Py_PrintSpecializationStats(1); + int res = _Py_PrintSpecializationStats(1); _Py_StatsClear(); - Py_RETURN_NONE; + return res; } +#endif // Py_STATS -#endif #ifdef ANDROID_API_LEVEL /*[clinic input] From db1ee6a19ab62191c16ecb732cb4dcaede98a902 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Sep 2023 20:09:21 +0200 Subject: [PATCH 524/598] gh-108740: Fix "make regen-all" race condition (#108741) Fix a race condition in "make regen-all". The deepfreeze.c source and files generated by Argument Clinic are now generated or updated before generating "global objects". Previously, some identifiers may miss depending on the order in which these files were generated. * "make regen-global-objects": Make sure that deepfreeze.c is generated and up to date, and always run "make clinic". * "make clinic" no longer runs generate_global_objects.py script. * "make regen-deepfreeze" now only updates deepfreeze.c (C file). It doesn't build deepfreeze.o (object) anymore. * Remove misleading messages in "make regen-global-objects" and "make clinic". They are now outdated, these commands are now safe to use. * Document generates files in Doc/using/configure.rst. Co-authored-by: Erlend E. Aasland --- Doc/using/configure.rst | 23 +++++++++++++++++++ Makefile.pre.in | 19 +++++++-------- ...-09-01-01-39-26.gh-issue-108740.JHExAQ.rst | 4 ++++ Tools/build/freeze_modules.py | 2 +- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index ad58255c33b33b..ba1898e35b5fbc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -60,6 +60,29 @@ See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". +Generated files +=============== + +To reduce build dependencies, Python source code contains multiple generated +files. Commands to regenerate all generated files:: + + make regen-all + make regen-stdlib-module-names + make regen-limited-abi + make regen-configure + +The ``Makefile.pre.in`` file documents generated files, their inputs, and tools used +to regenerate them. Search for ``regen-*`` make targets. + +The ``make regen-configure`` command runs `tiran/cpython_autoconf +`_ container for reproducible build; +see container ``entry.sh`` script. The container is optional, the following +command can be run locally, the generated files depend on autoconf and aclocal +versions:: + + autoreconf -ivf -Werror + + .. _configure-options: Configure Options diff --git a/Makefile.pre.in b/Makefile.pre.in index cf77cbd13ad4d9..aa3eaabc7559f6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -490,6 +490,7 @@ OBJECT_OBJS= \ Objects/weakrefobject.o \ @PERF_TRAMPOLINE_OBJ@ +DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c DEEPFREEZE_OBJS = Python/deepfreeze/deepfreeze.o ########################################################################## @@ -777,7 +778,6 @@ coverage-report: regen-token regen-frozen .PHONY: clinic clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py .PHONY: clinic-tests clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c @@ -1252,12 +1252,12 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) # Deepfreeze targets .PHONY: regen-deepfreeze -regen-deepfreeze: $(DEEPFREEZE_OBJS) +regen-deepfreeze: $(DEEPFREEZE_C) DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) # BEGIN: deepfreeze modules -Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) +$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS) $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ @@ -1284,8 +1284,6 @@ Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) Python/frozen_modules/frozen_only.h:frozen_only \ -o Python/deepfreeze/deepfreeze.c # END: deepfreeze modules - @echo "Note: Deepfreeze may have added some global objects," - @echo " so run 'make regen-global-objects' if necessary." # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib @@ -1294,11 +1292,12 @@ regen-importlib: regen-frozen ############################################################################ # Global objects +# Dependencies which can add and/or remove _Py_ID() identifiers: +# - deepfreeze.c +# - "make clinic" .PHONY: regen-global-objects -regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py +regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py $(DEEPFREEZE_C) clinic $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py - @echo "Note: Global objects can be added or removed by other tools (e.g. deepfreeze), " - @echo " so be sure to re-run regen-global-objects after those tools." ############################################################################ # ABI @@ -1320,9 +1319,10 @@ regen-limited-abi: all ############################################################################ # Regenerate all generated files +# "clinic" is regenerated implicitly via "regen-global-objects". .PHONY: regen-all regen-all: regen-cases regen-typeslots \ - regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ + regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @echo @@ -2597,6 +2597,7 @@ recheck: autoconf: (cd $(srcdir); autoreconf -ivf -Werror) +# See https://github.com/tiran/cpython_autoconf container .PHONY: regen-configure regen-configure: @if command -v podman >/dev/null; then RUNTIME="podman"; else RUNTIME="docker"; fi; \ diff --git a/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst b/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst new file mode 100644 index 00000000000000..190d50387f339e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-09-01-01-39-26.gh-issue-108740.JHExAQ.rst @@ -0,0 +1,4 @@ +Fix a race condition in ``make regen-all``. The ``deepfreeze.c`` source and +files generated by Argument Clinic are now generated or updated before +generating "global objects". Previously, some identifiers may miss depending +on the order in which these files were generated. Patch by Victor Stinner. diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index ee4dd2f8682ba4..12200979fa4bc7 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -585,7 +585,7 @@ def regen_makefile(modules): pyfiles = [] frozenfiles = [] rules = [''] - deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)", + deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)", "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) From bf414b7fcb7c8ba780a5e1d9f320ecef0c7f9488 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Sep 2023 22:02:01 +0300 Subject: [PATCH 525/598] C API tests: use special markers to test that output parameters were set (GH-109014) --- Modules/_testcapi/abstract.c | 8 +++---- Modules/_testcapi/code.c | 4 +++- Modules/_testcapi/dict.c | 8 ++++--- Modules/_testcapi/exceptions.c | 11 ++++++---- Modules/_testcapi/unicode.c | 38 +++++++++++++++++++++------------- Modules/_testcapi/util.h | 7 +++++++ Modules/_testcapimodule.c | 14 +++++++++---- 7 files changed, 60 insertions(+), 30 deletions(-) diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index 91a1ee3aaafc02..bde0d2848954ed 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -32,7 +32,7 @@ object_getattrstring(PyObject *self, PyObject *args) static PyObject * object_getoptionalattr(PyObject *self, PyObject *args) { - PyObject *obj, *attr_name, *value; + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } @@ -57,7 +57,7 @@ object_getoptionalattr(PyObject *self, PyObject *args) static PyObject * object_getoptionalattrstring(PyObject *self, PyObject *args) { - PyObject *obj, *value; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { @@ -207,7 +207,7 @@ mapping_getitemstring(PyObject *self, PyObject *args) static PyObject * mapping_getoptionalitem(PyObject *self, PyObject *args) { - PyObject *obj, *attr_name, *value; + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } @@ -232,7 +232,7 @@ mapping_getoptionalitem(PyObject *self, PyObject *args) static PyObject * mapping_getoptionalitemstring(PyObject *self, PyObject *args) { - PyObject *obj, *value; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { diff --git a/Modules/_testcapi/code.c b/Modules/_testcapi/code.c index 691dd5fe043811..c0193489b6f340 100644 --- a/Modules/_testcapi/code.c +++ b/Modules/_testcapi/code.c @@ -1,4 +1,5 @@ #include "parts.h" +#include "util.h" static Py_ssize_t get_code_extra_index(PyInterpreterState* interp) { @@ -75,7 +76,7 @@ test_code_extra(PyObject* self, PyObject *Py_UNUSED(callable)) } // Check the value is initially NULL - void *extra; + void *extra = UNINITIALIZED_PTR; int res = PyUnstable_Code_GetExtra(test_func_code, code_extra_index, &extra); if (res < 0) { goto finally; @@ -88,6 +89,7 @@ test_code_extra(PyObject* self, PyObject *Py_UNUSED(callable)) goto finally; } // Assert it was set correctly + extra = UNINITIALIZED_PTR; res = PyUnstable_Code_GetExtra(test_func_code, code_extra_index, &extra); if (res < 0) { goto finally; diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 6720f0437401ef..810989fbed85f9 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -139,7 +139,7 @@ dict_getitemwitherror(PyObject *self, PyObject *args) static PyObject * dict_getitemref(PyObject *self, PyObject *args) { - PyObject *obj, *attr_name, *value; + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } @@ -164,7 +164,7 @@ dict_getitemref(PyObject *self, PyObject *args) static PyObject * dict_getitemstringref(PyObject *self, PyObject *args) { - PyObject *obj, *value; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { @@ -276,7 +276,7 @@ dict_items(PyObject *self, PyObject *obj) static PyObject * dict_next(PyObject *self, PyObject *args) { - PyObject *mapping, *key, *value; + PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; Py_ssize_t pos; if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { return NULL; @@ -286,6 +286,8 @@ dict_next(PyObject *self, PyObject *args) if (rc != 0) { return Py_BuildValue("inOO", rc, pos, key, value); } + assert(key == UNINITIALIZED_PTR); + assert(value == UNINITIALIZED_PTR); if (PyErr_Occurred()) { return NULL; } diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index b1388d75711774..b54ce0cbb0dd20 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -120,12 +120,15 @@ _testcapi_exc_set_object_fetch_impl(PyObject *module, PyObject *exc, PyObject *obj) /*[clinic end generated code: output=7a5ff5f6d3cf687f input=77ec686f1f95fa38]*/ { - PyObject *type; - PyObject *value; - PyObject *tb; + PyObject *type = UNINITIALIZED_PTR; + PyObject *value = UNINITIALIZED_PTR; + PyObject *tb = UNINITIALIZED_PTR; PyErr_SetObject(exc, obj); PyErr_Fetch(&type, &value, &tb); + assert(type != UNINITIALIZED_PTR); + assert(value != UNINITIALIZED_PTR); + assert(tb != UNINITIALIZED_PTR); Py_XDECREF(type); Py_XDECREF(tb); return value; @@ -244,7 +247,7 @@ _testcapi_set_exc_info_impl(PyObject *module, PyObject *new_type, PyObject *new_value, PyObject *new_tb) /*[clinic end generated code: output=b55fa35dec31300e input=ea9f19e0f55fe5b3]*/ { - PyObject *type, *value, *tb; + PyObject *type = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR, *tb = UNINITIALIZED_PTR; PyErr_GetExcInfo(&type, &value, &tb); Py_INCREF(new_type); diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index f2cf9a744007d9..232b2ad543fca0 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -490,7 +490,7 @@ static PyObject * unicode_aswidecharstring(PyObject *self, PyObject *args) { PyObject *unicode, *result; - Py_ssize_t size = 100; + Py_ssize_t size = UNINITIALIZED_SIZE; wchar_t *buffer; if (!PyArg_ParseTuple(args, "O", &unicode)) @@ -498,8 +498,10 @@ unicode_aswidecharstring(PyObject *self, PyObject *args) NULLABLE(unicode); buffer = PyUnicode_AsWideCharString(unicode, &size); - if (buffer == NULL) + if (buffer == NULL) { + assert(size == UNINITIALIZED_SIZE); return NULL; + } result = PyUnicode_FromWideChar(buffer, size + 1); PyMem_Free(buffer); @@ -624,15 +626,17 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) PyObject *unicode; Py_ssize_t buflen; const char *s; - Py_ssize_t size = -100; + Py_ssize_t size = UNINITIALIZED_SIZE; if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; NULLABLE(unicode); s = PyUnicode_AsUTF8AndSize(unicode, &size); - if (s == NULL) + if (s == NULL) { + assert(size == UNINITIALIZED_SIZE); return NULL; + } return Py_BuildValue("(y#n)", s, buflen, size); } @@ -726,7 +730,7 @@ unicode_decodeutf7stateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - Py_ssize_t consumed; + Py_ssize_t consumed = UNINITIALIZED_SIZE; PyObject *result; if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) @@ -734,6 +738,7 @@ unicode_decodeutf7stateful(PyObject *self, PyObject *args) result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); if (!result) { + assert(consumed == UNINITIALIZED_SIZE); return NULL; } return Py_BuildValue("(Nn)", result, consumed); @@ -760,7 +765,7 @@ unicode_decodeutf8stateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - Py_ssize_t consumed = 123456789; + Py_ssize_t consumed = UNINITIALIZED_SIZE; PyObject *result; if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) @@ -768,6 +773,7 @@ unicode_decodeutf8stateful(PyObject *self, PyObject *args) result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); if (!result) { + assert(consumed == UNINITIALIZED_SIZE); return NULL; } return Py_BuildValue("(Nn)", result, consumed); @@ -788,7 +794,7 @@ unicode_decodeutf32(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - int byteorder; + int byteorder = UNINITIALIZED_INT; PyObject *result; if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) @@ -808,8 +814,8 @@ unicode_decodeutf32stateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - int byteorder; - Py_ssize_t consumed; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; PyObject *result; if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) @@ -817,6 +823,7 @@ unicode_decodeutf32stateful(PyObject *self, PyObject *args) result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); if (!result) { + assert(consumed == UNINITIALIZED_SIZE); return NULL; } return Py_BuildValue("(iNn)", byteorder, result, consumed); @@ -837,7 +844,7 @@ unicode_decodeutf16(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - int byteorder = 0; + int byteorder = UNINITIALIZED_INT; PyObject *result; if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) @@ -857,8 +864,8 @@ unicode_decodeutf16stateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - int byteorder; - Py_ssize_t consumed; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; PyObject *result; if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) @@ -866,6 +873,7 @@ unicode_decodeutf16stateful(PyObject *self, PyObject *args) result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); if (!result) { + assert(consumed == UNINITIALIZED_SIZE); return NULL; } return Py_BuildValue("(iNn)", byteorder, result, consumed); @@ -1019,7 +1027,7 @@ unicode_decodembcsstateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - Py_ssize_t consumed; + Py_ssize_t consumed = UNINITIALIZED_SIZE; PyObject *result; if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) @@ -1027,6 +1035,7 @@ unicode_decodembcsstateful(PyObject *self, PyObject *args) result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); if (!result) { + assert(consumed == UNINITIALIZED_SIZE); return NULL; } return Py_BuildValue("(Nn)", result, consumed); @@ -1040,7 +1049,7 @@ unicode_decodecodepagestateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - Py_ssize_t consumed; + Py_ssize_t consumed = UNINITIALIZED_SIZE; PyObject *result; if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) @@ -1048,6 +1057,7 @@ unicode_decodecodepagestateful(PyObject *self, PyObject *args) result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); if (!result) { + assert(consumed == UNINITIALIZED_SIZE); return NULL; } return Py_BuildValue("(Nn)", result, consumed); diff --git a/Modules/_testcapi/util.h b/Modules/_testcapi/util.h index 05ccb12a4444f8..4cf7e226382bfc 100644 --- a/Modules/_testcapi/util.h +++ b/Modules/_testcapi/util.h @@ -23,3 +23,10 @@ assert(!PyErr_Occurred()); \ return PyLong_FromSsize_t(_ret); \ } while (0) + +/* Marker to check that pointer value was set. */ +#define UNINITIALIZED_PTR ((void *)"uninitialized") +/* Marker to check that Py_ssize_t value was set. */ +#define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) +/* Marker to check that integer value was set. */ +#define UNINITIALIZED_INT (63256717) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ab33702cdfd872..5ff5172f4f30eb 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -217,10 +217,13 @@ test_dict_inner(int count) Py_DECREF(v); } + k = v = UNINITIALIZED_PTR; while (PyDict_Next(dict, &pos, &k, &v)) { PyObject *o; iterations++; + assert(k != UNINITIALIZED_PTR); + assert(v != UNINITIALIZED_PTR); i = PyLong_AS_LONG(v) + 1; o = PyLong_FromLong(i); if (o == NULL) @@ -230,7 +233,10 @@ test_dict_inner(int count) return -1; } Py_DECREF(o); + k = v = UNINITIALIZED_PTR; } + assert(k == UNINITIALIZED_PTR); + assert(v == UNINITIALIZED_PTR); Py_DECREF(dict); @@ -3118,7 +3124,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(Py_REFCNT(obj) == refcnt); // test PyWeakref_GetRef(), reference is alive - PyObject *ref = Py_True; // marker to check that value was set + PyObject *ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(weakref, &ref) == 1); assert(ref == obj); assert(Py_REFCNT(obj) == (refcnt + 1)); @@ -3140,7 +3146,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(PyWeakref_GET_OBJECT(weakref) == Py_None); // test PyWeakref_GetRef(), reference is dead - ref = Py_True; + ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(weakref, &ref) == 0); assert(ref == NULL); @@ -3152,7 +3158,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) // test PyWeakref_GetRef(), invalid type assert(!PyErr_Occurred()); - ref = Py_True; + ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(invalid_weakref, &ref) == -1); assert(PyErr_ExceptionMatches(PyExc_TypeError)); PyErr_Clear(); @@ -3164,7 +3170,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyErr_Clear(); // test PyWeakref_GetRef(NULL) - ref = Py_True; // marker to check that value was set + ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(NULL, &ref) == -1); assert(PyErr_ExceptionMatches(PyExc_SystemError)); assert(ref == NULL); From 3a08db8d137a552aebf678bcd2e7444ab72e62c5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Sep 2023 22:47:38 +0300 Subject: [PATCH 526/598] gh-106307: Fix PyMapping_GetOptionalItemString() (GH-108797) The resulting pointer was not set to NULL if the creation of a temporary string object was failed. The tests were also missed due to oversight. --- Lib/test/test_capi/test_abstract.py | 37 +++++++++++++++++++++++++++++ Objects/abstract.c | 2 ++ 2 files changed, 39 insertions(+) diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 3f51e5b28104d9..671f62b977d2aa 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -265,6 +265,43 @@ def test_mapping_getitemstring(self): self.assertRaises(TypeError, getitemstring, [], b'a') self.assertRaises(SystemError, getitemstring, NULL, b'a') + def test_mapping_getoptionalitem(self): + getitem = _testcapi.mapping_getoptionalitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertEqual(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertEqual(getitem(dct2, 'b'), KeyError) + + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') + + self.assertRaises(TypeError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(IndexError, getitem, [], 1) + self.assertRaises(TypeError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_mapping_getoptionalitemstring(self): + getitemstring = _testcapi.mapping_getoptionalitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertEqual(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertEqual(getitemstring(dct2, b'b'), KeyError) + + self.assertRaises(TypeError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, {}, NULL) + self.assertRaises(TypeError, getitemstring, [], b'a') + # CRASHES getitemstring(NULL, b'a') + def test_mapping_haskey(self): haskey = _testcapi.mapping_haskey dct = {'a': 1, '\U0001f40d': 2} diff --git a/Objects/abstract.c b/Objects/abstract.c index 6713926b86d218..b57190d2521e11 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2393,11 +2393,13 @@ int PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) { if (key == NULL) { + *result = NULL; null_error(); return -1; } PyObject *okey = PyUnicode_FromString(key); if (okey == NULL) { + *result = NULL; return -1; } int rc = PyMapping_GetOptionalItem(obj, okey, result); From f9bd6e49ae58e0ba2934f29dd0f3299ba844cc8d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 7 Sep 2023 06:28:00 +1000 Subject: [PATCH 527/598] GH-90690: Mention removal of ``PRECALL`` in What's New (#103910) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Hugo van Kemenade --- Doc/whatsnew/3.12.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index acdfe9e8b19a4f..3afcd1355db47c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -997,6 +997,9 @@ CPython bytecode changes * Remove the :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` instructions. (Contributed by Irit Katriel in :gh:`102859`.) +* Removed the :opcode:`!PRECALL` instruction. (Contributed by Mark Shannon in + :gh:`92925`.) + * Add the :opcode:`LOAD_FAST_AND_CLEAR` instruction as part of the implementation of :pep:`709`. (Contributed by Carl Meyer in :gh:`101441`.) From 9f0c0a46f00d687e921990ee83894b2f4ce8a6e7 Mon Sep 17 00:00:00 2001 From: Shahriar Heidrich Date: Wed, 6 Sep 2023 22:53:32 +0200 Subject: [PATCH 528/598] gh-107732: Mention dir support in importlib.resources docs (#107734) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Hugo van Kemenade --- Doc/library/importlib.resources.rst | 13 ++++++++----- Doc/whatsnew/3.12.rst | 6 ++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 76faf731144779..3de97e80311a17 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -82,15 +82,18 @@ for example, a package and its resources can be imported from a zip file using .. function:: as_file(traversable) Given a :class:`~importlib.resources.abc.Traversable` object representing - a file, typically from :func:`importlib.resources.files`, return - a context manager for use in a :keyword:`with` statement. + a file or directory, typically from :func:`importlib.resources.files`, + return a context manager for use in a :keyword:`with` statement. The context manager provides a :class:`pathlib.Path` object. - Exiting the context manager cleans up any temporary file created when the - resource was extracted from e.g. a zip file. + Exiting the context manager cleans up any temporary file or directory + created when the resource was extracted from e.g. a zip file. Use ``as_file`` when the Traversable methods - (``read_text``, etc) are insufficient and an actual file on + (``read_text``, etc) are insufficient and an actual file or directory on the file system is required. .. versionadded:: 3.9 + + .. versionchanged:: 3.12 + Added support for ``traversable`` representing a directory. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3afcd1355db47c..bdccbe1012c9e2 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -630,6 +630,12 @@ fractions * Objects of type :class:`fractions.Fraction` now support float-style formatting. (Contributed by Mark Dickinson in :gh:`100161`.) +importlib.resources +------------------- + +* :func:`importlib.resources.as_file` now supports resource directories. + (Contributed by Jason R. Coombs in :gh:`97930`.) + inspect ------- From 6f3c138dfa868b32d3288898923bbfa388f2fa5d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Sep 2023 23:55:42 +0300 Subject: [PATCH 529/598] gh-108751: Add copy.replace() function (GH-108752) It creates a modified copy of an object by calling the object's __replace__() method. It is a generalization of dataclasses.replace(), named tuple's _replace() method and replace() methods in various classes, and supports all these stdlib classes. --- Doc/library/collections.rst | 2 + Doc/library/copy.rst | 30 +++- Doc/library/dataclasses.rst | 2 + Doc/library/datetime.rst | 9 ++ Doc/library/inspect.rst | 11 +- Doc/library/types.rst | 2 + Doc/whatsnew/3.13.rst | 12 ++ Lib/_pydatetime.py | 6 + Lib/collections/__init__.py | 1 + Lib/copy.py | 13 ++ Lib/dataclasses.py | 9 +- Lib/inspect.py | 4 + Lib/test/datetimetester.py | 137 +++++++++++------- Lib/test/test_code.py | 7 + Lib/test/test_copy.py | 66 ++++++++- Lib/test/test_inspect.py | 65 ++++++++- ...-09-01-13-14-08.gh-issue-108751.2itqwe.rst | 2 + Modules/_datetimemodule.c | 6 + Objects/codeobject.c | 1 + 19 files changed, 314 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-01-13-14-08.gh-issue-108751.2itqwe.rst diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index b8b231bb15b1b0..03cb1dca8f816c 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -979,6 +979,8 @@ field names, the method and attribute names start with an underscore. >>> for partnum, record in inventory.items(): ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) + Named tuples are also supported by generic function :func:`copy.replace`. + .. attribute:: somenamedtuple._fields Tuple of strings listing the field names. Useful for introspection diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 8f32477ed508c3..cc4ca034d07a00 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -17,14 +17,22 @@ operations (explained below). Interface summary: -.. function:: copy(x) +.. function:: copy(obj) - Return a shallow copy of *x*. + Return a shallow copy of *obj*. -.. function:: deepcopy(x[, memo]) +.. function:: deepcopy(obj[, memo]) - Return a deep copy of *x*. + Return a deep copy of *obj*. + + +.. function:: replace(obj, /, **changes) + + Creates a new object of the same type as *obj*, replacing fields with values + from *changes*. + + .. versionadded:: 3.13 .. exception:: Error @@ -89,6 +97,20 @@ with the component as first argument and the memo dictionary as second argument. The memo dictionary should be treated as an opaque object. +.. index:: + single: __replace__() (replace protocol) + +Function :func:`replace` is more limited than :func:`copy` and :func:`deepcopy`, +and only supports named tuples created by :func:`~collections.namedtuple`, +:mod:`dataclasses`, and other classes which define method :meth:`!__replace__`. + + .. method:: __replace__(self, /, **changes) + :noindex: + +:meth:`!__replace__` should create a new object of the same type, +replacing fields with values from *changes*. + + .. seealso:: Module :mod:`pickle` diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index d68748767c5e37..d78a6071a50e4b 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -456,6 +456,8 @@ Module contents ``replace()`` (or similarly named) method which handles instance copying. + Dataclass instances are also supported by generic function :func:`copy.replace`. + .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass or an instance of one, diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 04cc75562937e0..0b9d42f32e3bd6 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -652,6 +652,9 @@ Instance methods: >>> d.replace(day=26) datetime.date(2002, 12, 26) + :class:`date` objects are also supported by generic function + :func:`copy.replace`. + .. method:: date.timetuple() @@ -1251,6 +1254,9 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. + :class:`datetime` objects are also supported by generic function + :func:`copy.replace`. + .. versionadded:: 3.6 Added the ``fold`` argument. @@ -1827,6 +1833,9 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. + :class:`time` objects are also supported by generic function + :func:`copy.replace`. + .. versionadded:: 3.6 Added the ``fold`` argument. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 603ac3263bb9ec..fe0ed135029f0f 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -689,8 +689,8 @@ function. The optional *return_annotation* argument, can be an arbitrary Python object, is the "return" annotation of the callable. - Signature objects are *immutable*. Use :meth:`Signature.replace` to make a - modified copy. + Signature objects are *immutable*. Use :meth:`Signature.replace` or + :func:`copy.replace` to make a modified copy. .. versionchanged:: 3.5 Signature objects are picklable and :term:`hashable`. @@ -746,6 +746,9 @@ function. >>> str(new_sig) "(a, b) -> 'new return anno'" + Signature objects are also supported by generic function + :func:`copy.replace`. + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globalns=None, localns=None) Return a :class:`Signature` (or its subclass) object for a given callable @@ -769,7 +772,7 @@ function. .. class:: Parameter(name, kind, *, default=Parameter.empty, annotation=Parameter.empty) Parameter objects are *immutable*. Instead of modifying a Parameter object, - you can use :meth:`Parameter.replace` to create a modified copy. + you can use :meth:`Parameter.replace` or :func:`copy.replace` to create a modified copy. .. versionchanged:: 3.5 Parameter objects are picklable and :term:`hashable`. @@ -892,6 +895,8 @@ function. >>> str(param.replace(default=Parameter.empty, annotation='spam')) "foo:'spam'" + Parameter objects are also supported by generic function :func:`copy.replace`. + .. versionchanged:: 3.4 In Python 3.3 Parameter objects were allowed to have ``name`` set to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 8cbe17df16f107..82300afef0641e 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -200,6 +200,8 @@ Standard names are defined for the following types: Return a copy of the code object with new values for the specified fields. + Code objects are also supported by generic function :func:`copy.replace`. + .. versionadded:: 3.8 .. data:: CellType diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index de23172ac7a43b..8c6467562aeb62 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -115,6 +115,18 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) +copy +---- + +* Add :func:`copy.replace` function which allows to create a modified copy of + an object, which is especially usefule for immutable objects. + It supports named tuples created with the factory function + :func:`collections.namedtuple`, :class:`~dataclasses.dataclass` instances, + various :mod:`datetime` objects, :class:`~inspect.Signature` objects, + :class:`~inspect.Parameter` objects, :ref:`code object `, and + any user classes which define the :meth:`!__replace__` method. + (Contributed by Serhiy Storchaka in :gh:`108751`.) + dbm --- diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 549fcda19dccf2..df616bbaf8388d 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1112,6 +1112,8 @@ def replace(self, year=None, month=None, day=None): day = self._day return type(self)(year, month, day) + __replace__ = replace + # Comparisons of date objects with other. def __eq__(self, other): @@ -1637,6 +1639,8 @@ def replace(self, hour=None, minute=None, second=None, microsecond=None, fold = self._fold return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + # Pickle support. def _getstate(self, protocol=3): @@ -1983,6 +1987,8 @@ def replace(self, year=None, month=None, day=None, hour=None, return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + def _local_timezone(self): if self.tzinfo is None: ts = self._mktime() diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 8652dc8a4ec450..a461550ea40da7 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -495,6 +495,7 @@ def __getnewargs__(self): '_field_defaults': field_defaults, '__new__': __new__, '_make': _make, + '__replace__': _replace, '_replace': _replace, '__repr__': __repr__, '_asdict': _asdict, diff --git a/Lib/copy.py b/Lib/copy.py index da2908ef623d8c..6d7bb9a111b5b4 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -290,3 +290,16 @@ def _reconstruct(x, memo, func, args, return y del types, weakref + + +def replace(obj, /, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for immutable objects, like named tuples or + frozen dataclasses. + """ + cls = obj.__class__ + func = getattr(cls, '__replace__', None) + if func is None: + raise TypeError(f"replace() does not support {cls.__name__} objects") + return func(obj, **changes) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 21f3fa5c213f1f..84f8d68ce092a4 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1073,6 +1073,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, globals, slots, )) + _set_new_attribute(cls, '__replace__', _replace) # Get the fields as a list, and include only real fields. This is # used in all of the following methods. @@ -1546,13 +1547,15 @@ class C: c1 = replace(c, x=3) assert c1.x == 3 and c1.y == 2 """ + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + return _replace(obj, **changes) + +def _replace(obj, /, **changes): # We're going to mutate 'changes', but that's okay because it's a # new dict, even if called with 'replace(obj, **my_changes)'. - if not _is_dataclass_instance(obj): - raise TypeError("replace() should be called on dataclass instances") - # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. diff --git a/Lib/inspect.py b/Lib/inspect.py index c8211833dd0831..aaa22bef896602 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2870,6 +2870,8 @@ def __str__(self): return formatted + __replace__ = replace + def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) @@ -3130,6 +3132,8 @@ def replace(self, *, parameters=_void, return_annotation=_void): return type(self)(parameters, return_annotation=return_annotation) + __replace__ = replace + def _hash_basis(self): params = tuple(param for param in self.parameters.values() if param.kind != _KEYWORD_ONLY) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 55e061950ff280..8bda17358db87f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1699,22 +1699,23 @@ def test_replace(self): cls = self.theclass args = [1, 2, 3] base = cls(*args) - self.assertEqual(base, base.replace()) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) - i = 0 - for name, newval in (("year", 2), - ("month", 3), - ("day", 4)): + changes = (("year", 2), + ("month", 3), + ("day", 4)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) def test_subclass_replace(self): class DateSubclass(self.theclass): @@ -1722,6 +1723,7 @@ class DateSubclass(self.theclass): dt = DateSubclass(2012, 1, 1) self.assertIs(type(dt.replace(year=2013)), DateSubclass) + self.assertIs(type(copy.replace(dt, year=2013)), DateSubclass) def test_subclass_date(self): @@ -2856,26 +2858,27 @@ def test_replace(self): cls = self.theclass args = [1, 2, 3, 4, 5, 6, 7] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("year", 2), - ("month", 3), - ("day", 4), - ("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4), + ("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) @support.run_with_tz('EDT4') def test_astimezone(self): @@ -3671,19 +3674,19 @@ def test_replace(self): cls = self.theclass args = [1, 2, 3, 4] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Out of bounds. base = cls(1) @@ -3691,6 +3694,10 @@ def test_replace(self): self.assertRaises(ValueError, base.replace, minute=-1) self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) + self.assertRaises(ValueError, copy.replace, base, hour=24) + self.assertRaises(ValueError, copy.replace, base, minute=-1) + self.assertRaises(ValueError, copy.replace, base, second=100) + self.assertRaises(ValueError, copy.replace, base, microsecond=1000000) def test_subclass_replace(self): class TimeSubclass(self.theclass): @@ -3698,6 +3705,7 @@ class TimeSubclass(self.theclass): ctime = TimeSubclass(12, 30) self.assertIs(type(ctime.replace(hour=10)), TimeSubclass) + self.assertIs(type(copy.replace(ctime, hour=10)), TimeSubclass) def test_subclass_time(self): @@ -4085,31 +4093,37 @@ def test_replace(self): zm200 = FixedOffset(timedelta(minutes=-200), "-200") args = [1, 2, 3, 4, z100] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8), - ("tzinfo", zm200)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8), + ("tzinfo", zm200)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Ensure we can get rid of a tzinfo. self.assertEqual(base.tzname(), "+100") base2 = base.replace(tzinfo=None) self.assertIsNone(base2.tzinfo) self.assertIsNone(base2.tzname()) + base22 = copy.replace(base, tzinfo=None) + self.assertIsNone(base22.tzinfo) + self.assertIsNone(base22.tzname()) # Ensure we can add one. base3 = base2.replace(tzinfo=z100) self.assertEqual(base, base3) self.assertIs(base.tzinfo, base3.tzinfo) + base32 = copy.replace(base22, tzinfo=z100) + self.assertEqual(base, base32) + self.assertIs(base.tzinfo, base32.tzinfo) # Out of bounds. base = cls(1) @@ -4117,6 +4131,10 @@ def test_replace(self): self.assertRaises(ValueError, base.replace, minute=-1) self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) + self.assertRaises(ValueError, copy.replace, base, hour=24) + self.assertRaises(ValueError, copy.replace, base, minute=-1) + self.assertRaises(ValueError, copy.replace, base, second=100) + self.assertRaises(ValueError, copy.replace, base, microsecond=1000000) def test_mixed_compare(self): t1 = self.theclass(1, 2, 3) @@ -4885,38 +4903,45 @@ def test_replace(self): zm200 = FixedOffset(timedelta(minutes=-200), "-200") args = [1, 2, 3, 4, 5, 6, 7, z100] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("year", 2), - ("month", 3), - ("day", 4), - ("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8), - ("tzinfo", zm200)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4), + ("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8), + ("tzinfo", zm200)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Ensure we can get rid of a tzinfo. self.assertEqual(base.tzname(), "+100") base2 = base.replace(tzinfo=None) self.assertIsNone(base2.tzinfo) self.assertIsNone(base2.tzname()) + base22 = copy.replace(base, tzinfo=None) + self.assertIsNone(base22.tzinfo) + self.assertIsNone(base22.tzname()) # Ensure we can add one. base3 = base2.replace(tzinfo=z100) self.assertEqual(base, base3) self.assertIs(base.tzinfo, base3.tzinfo) + base32 = copy.replace(base22, tzinfo=z100) + self.assertEqual(base, base32) + self.assertIs(base.tzinfo, base32.tzinfo) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) def test_more_astimezone(self): # The inherited test_astimezone covered some trivial and error cases. diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index e056c16466e8c4..812c0682569207 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -125,6 +125,7 @@ """ +import copy import inspect import sys import threading @@ -280,11 +281,17 @@ def func2(): with self.subTest(attr=attr, value=value): new_code = code.replace(**{attr: value}) self.assertEqual(getattr(new_code, attr), value) + new_code = copy.replace(code, **{attr: value}) + self.assertEqual(getattr(new_code, attr), value) new_code = code.replace(co_varnames=code2.co_varnames, co_nlocals=code2.co_nlocals) self.assertEqual(new_code.co_varnames, code2.co_varnames) self.assertEqual(new_code.co_nlocals, code2.co_nlocals) + new_code = copy.replace(code, co_varnames=code2.co_varnames, + co_nlocals=code2.co_nlocals) + self.assertEqual(new_code.co_varnames, code2.co_varnames) + self.assertEqual(new_code.co_nlocals, code2.co_nlocals) def test_nlocals_mismatch(self): def func(): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 826e46824e004c..c66c6eeb00811e 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -4,7 +4,7 @@ import copyreg import weakref import abc -from operator import le, lt, ge, gt, eq, ne +from operator import le, lt, ge, gt, eq, ne, attrgetter import unittest from test import support @@ -899,7 +899,71 @@ def m(self): g.b() +class TestReplace(unittest.TestCase): + + def test_unsupported(self): + self.assertRaises(TypeError, copy.replace, 1) + self.assertRaises(TypeError, copy.replace, []) + self.assertRaises(TypeError, copy.replace, {}) + def f(): pass + self.assertRaises(TypeError, copy.replace, f) + class A: pass + self.assertRaises(TypeError, copy.replace, A) + self.assertRaises(TypeError, copy.replace, A()) + + def test_replace_method(self): + class A: + def __new__(cls, x, y=0): + self = object.__new__(cls) + self.x = x + self.y = y + return self + + def __init__(self, *args, **kwargs): + self.z = self.x + self.y + + def __replace__(self, **changes): + x = changes.get('x', self.x) + y = changes.get('y', self.y) + return type(self)(x, y) + + attrs = attrgetter('x', 'y', 'z') + a = A(11, 22) + self.assertEqual(attrs(copy.replace(a)), (11, 22, 33)) + self.assertEqual(attrs(copy.replace(a, x=1)), (1, 22, 23)) + self.assertEqual(attrs(copy.replace(a, y=2)), (11, 2, 13)) + self.assertEqual(attrs(copy.replace(a, x=1, y=2)), (1, 2, 3)) + + def test_namedtuple(self): + from collections import namedtuple + Point = namedtuple('Point', 'x y', defaults=(0,)) + p = Point(11, 22) + self.assertEqual(copy.replace(p), (11, 22)) + self.assertEqual(copy.replace(p, x=1), (1, 22)) + self.assertEqual(copy.replace(p, y=2), (11, 2)) + self.assertEqual(copy.replace(p, x=1, y=2), (1, 2)) + with self.assertRaisesRegex(ValueError, 'unexpected field name'): + copy.replace(p, x=1, error=2) + + def test_dataclass(self): + from dataclasses import dataclass + @dataclass + class C: + x: int + y: int = 0 + + attrs = attrgetter('x', 'y') + c = C(11, 22) + self.assertEqual(attrs(copy.replace(c)), (11, 22)) + self.assertEqual(attrs(copy.replace(c, x=1)), (1, 22)) + self.assertEqual(attrs(copy.replace(c, y=2)), (11, 2)) + self.assertEqual(attrs(copy.replace(c, x=1, y=2)), (1, 2)) + with self.assertRaisesRegex(TypeError, 'unexpected keyword argument'): + copy.replace(c, x=1, error=2) + + def global_foo(x, y): return x+y + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 78ef817906b2aa..2fb356a0529ab0 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1,6 +1,7 @@ import asyncio import builtins import collections +import copy import datetime import functools import importlib @@ -3830,6 +3831,28 @@ def test(a_po, /, *, b, **kwargs): P('bar', P.VAR_POSITIONAL)])), '(foo, /, *bar)') + def test_signature_replace_parameters(self): + def test(a, b) -> 42: + pass + + sig = inspect.signature(test) + parameters = sig.parameters + sig = sig.replace(parameters=list(parameters.values())[1:]) + self.assertEqual(list(sig.parameters), ['b']) + self.assertEqual(sig.parameters['b'], parameters['b']) + self.assertEqual(sig.return_annotation, 42) + sig = sig.replace(parameters=()) + self.assertEqual(dict(sig.parameters), {}) + + sig = inspect.signature(test) + parameters = sig.parameters + sig = copy.replace(sig, parameters=list(parameters.values())[1:]) + self.assertEqual(list(sig.parameters), ['b']) + self.assertEqual(sig.parameters['b'], parameters['b']) + self.assertEqual(sig.return_annotation, 42) + sig = copy.replace(sig, parameters=()) + self.assertEqual(dict(sig.parameters), {}) + def test_signature_replace_anno(self): def test() -> 42: pass @@ -3843,6 +3866,15 @@ def test() -> 42: self.assertEqual(sig.return_annotation, 42) self.assertEqual(sig, inspect.signature(test)) + sig = inspect.signature(test) + sig = copy.replace(sig, return_annotation=None) + self.assertIs(sig.return_annotation, None) + sig = copy.replace(sig, return_annotation=sig.empty) + self.assertIs(sig.return_annotation, sig.empty) + sig = copy.replace(sig, return_annotation=42) + self.assertEqual(sig.return_annotation, 42) + self.assertEqual(sig, inspect.signature(test)) + def test_signature_replaced(self): def test(): pass @@ -4187,41 +4219,66 @@ def test_signature_parameter_replace(self): p = inspect.Parameter('foo', default=42, kind=inspect.Parameter.KEYWORD_ONLY) - self.assertIsNot(p, p.replace()) - self.assertEqual(p, p.replace()) + self.assertIsNot(p.replace(), p) + self.assertEqual(p.replace(), p) + self.assertIsNot(copy.replace(p), p) + self.assertEqual(copy.replace(p), p) p2 = p.replace(annotation=1) self.assertEqual(p2.annotation, 1) p2 = p2.replace(annotation=p2.empty) - self.assertEqual(p, p2) + self.assertEqual(p2, p) + p3 = copy.replace(p, annotation=1) + self.assertEqual(p3.annotation, 1) + p3 = copy.replace(p3, annotation=p3.empty) + self.assertEqual(p3, p) p2 = p2.replace(name='bar') self.assertEqual(p2.name, 'bar') self.assertNotEqual(p2, p) + p3 = copy.replace(p3, name='bar') + self.assertEqual(p3.name, 'bar') + self.assertNotEqual(p3, p) with self.assertRaisesRegex(ValueError, 'name is a required attribute'): p2 = p2.replace(name=p2.empty) + with self.assertRaisesRegex(ValueError, + 'name is a required attribute'): + p3 = copy.replace(p3, name=p3.empty) p2 = p2.replace(name='foo', default=None) self.assertIs(p2.default, None) self.assertNotEqual(p2, p) + p3 = copy.replace(p3, name='foo', default=None) + self.assertIs(p3.default, None) + self.assertNotEqual(p3, p) p2 = p2.replace(name='foo', default=p2.empty) self.assertIs(p2.default, p2.empty) - + p3 = copy.replace(p3, name='foo', default=p3.empty) + self.assertIs(p3.default, p3.empty) p2 = p2.replace(default=42, kind=p2.POSITIONAL_OR_KEYWORD) self.assertEqual(p2.kind, p2.POSITIONAL_OR_KEYWORD) self.assertNotEqual(p2, p) + p3 = copy.replace(p3, default=42, kind=p3.POSITIONAL_OR_KEYWORD) + self.assertEqual(p3.kind, p3.POSITIONAL_OR_KEYWORD) + self.assertNotEqual(p3, p) with self.assertRaisesRegex(ValueError, "value " "is not a valid Parameter.kind"): p2 = p2.replace(kind=p2.empty) + with self.assertRaisesRegex(ValueError, + "value " + "is not a valid Parameter.kind"): + p3 = copy.replace(p3, kind=p3.empty) p2 = p2.replace(kind=p2.KEYWORD_ONLY) self.assertEqual(p2, p) + p3 = copy.replace(p3, kind=p3.KEYWORD_ONLY) + self.assertEqual(p3, p) def test_signature_parameter_positional_only(self): with self.assertRaisesRegex(TypeError, 'name must be a str'): diff --git a/Misc/NEWS.d/next/Library/2023-09-01-13-14-08.gh-issue-108751.2itqwe.rst b/Misc/NEWS.d/next/Library/2023-09-01-13-14-08.gh-issue-108751.2itqwe.rst new file mode 100644 index 00000000000000..7bc21fe6c81760 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-01-13-14-08.gh-issue-108751.2itqwe.rst @@ -0,0 +1,2 @@ +Add :func:`copy.replace` function which allows to create a modified copy of +an object. It supports named tuples, dataclasses, and many other objects. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 191db3f84088d5..0d356779cfe192 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3590,6 +3590,8 @@ static PyMethodDef date_methods[] = { {"replace", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return date with new specified fields.")}, + {"__replace__", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS}, + {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -4719,6 +4721,8 @@ static PyMethodDef time_methods[] = { {"replace", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return time with new specified fields.")}, + {"__replace__", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS}, + {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -6579,6 +6583,8 @@ static PyMethodDef datetime_methods[] = { {"replace", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return datetime with new specified fields.")}, + {"__replace__", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS}, + {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 70a0c2ebd66b24..58306075cad48b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2145,6 +2145,7 @@ static struct PyMethodDef code_methods[] = { {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF + {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS}, {NULL, NULL} /* sentinel */ }; From 6971e40c2e02181e486b52f4f6bf98709d35443c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 6 Sep 2023 13:59:50 -0700 Subject: [PATCH 530/598] GH-104584: Restore frame->stacktop on optimizer error (GH-108953) --- .../2023-09-05-11-31-27.gh-issue-104584.IRSXA2.rst | 2 ++ Python/optimizer.c | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-05-11-31-27.gh-issue-104584.IRSXA2.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-05-11-31-27.gh-issue-104584.IRSXA2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-05-11-31-27.gh-issue-104584.IRSXA2.rst new file mode 100644 index 00000000000000..7f556bf8c31c11 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-05-11-31-27.gh-issue-104584.IRSXA2.rst @@ -0,0 +1,2 @@ +Fix a crash when running with :envvar:`PYTHONUOPS` or :option:`-X uops <-X>` +enabled and an error occurs during optimization. diff --git a/Python/optimizer.c b/Python/optimizer.c index 8aaf9f9fd7cb14..c6d0f9e5bf22d7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -169,6 +169,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI if (err <= 0) { assert(executor == NULL); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); return NULL; } goto jump_to_destination; From f0f96a9f40762499811681d405b6f922b6ed7a55 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 6 Sep 2023 22:19:54 +0100 Subject: [PATCH 531/598] GH-108202: Document ``calendar``'s command-line interface (#109020) Co-authored-by: Hugo van Kemenade --- Doc/library/calendar.rst | 143 +++++++++++++++++++++++++++++++++++++++ Lib/calendar.py | 2 +- 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 1868eb1cd359c4..157a7537f97dc6 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -489,3 +489,146 @@ The :mod:`calendar` module defines the following exceptions: Module :mod:`time` Low-level time related functions. + + +.. _calendar-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 2.5 + +The :mod:`calendar` module can be executed as a script from the command line +to interactively print a calendar. + +.. code-block:: shell + + python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] + [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] + [year] [month] + + +For example, to print a calendar for the year 2000: + +.. code-block:: console + + $ python -m calendar 2000 + 2000 + + January February March + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 1 2 3 4 5 + 3 4 5 6 7 8 9 7 8 9 10 11 12 13 6 7 8 9 10 11 12 + 10 11 12 13 14 15 16 14 15 16 17 18 19 20 13 14 15 16 17 18 19 + 17 18 19 20 21 22 23 21 22 23 24 25 26 27 20 21 22 23 24 25 26 + 24 25 26 27 28 29 30 28 29 27 28 29 30 31 + 31 + + April May June + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 7 1 2 3 4 + 3 4 5 6 7 8 9 8 9 10 11 12 13 14 5 6 7 8 9 10 11 + 10 11 12 13 14 15 16 15 16 17 18 19 20 21 12 13 14 15 16 17 18 + 17 18 19 20 21 22 23 22 23 24 25 26 27 28 19 20 21 22 23 24 25 + 24 25 26 27 28 29 30 29 30 31 26 27 28 29 30 + + July August September + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 1 2 3 + 3 4 5 6 7 8 9 7 8 9 10 11 12 13 4 5 6 7 8 9 10 + 10 11 12 13 14 15 16 14 15 16 17 18 19 20 11 12 13 14 15 16 17 + 17 18 19 20 21 22 23 21 22 23 24 25 26 27 18 19 20 21 22 23 24 + 24 25 26 27 28 29 30 28 29 30 31 25 26 27 28 29 30 + 31 + + October November December + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 1 2 3 4 5 1 2 3 + 2 3 4 5 6 7 8 6 7 8 9 10 11 12 4 5 6 7 8 9 10 + 9 10 11 12 13 14 15 13 14 15 16 17 18 19 11 12 13 14 15 16 17 + 16 17 18 19 20 21 22 20 21 22 23 24 25 26 18 19 20 21 22 23 24 + 23 24 25 26 27 28 29 27 28 29 30 25 26 27 28 29 30 31 + 30 31 + + +The following options are accepted: + +.. program:: calendar + + +.. option:: --help, -h + + Show the help message and exit. + + +.. option:: --locale LOCALE, -L LOCALE + + The locale to use for month and weekday names. + Defaults to English. + + +.. option:: --encoding ENCODING, -e ENCODING + + The encoding to use for output. + :option:`--encoding` is required if :option:`--locale` is set. + + +.. option:: --type {text,html}, -t {text,html} + + Print the calendar to the terminal as text, + or as an HTML document. + + +.. option:: year + + The year to print the calendar for. + Must be a number between 1 and 9999. + Defaults to the current year. + + +.. option:: month + + The month of the specified :option:`year` to print the calendar for. + Must be a number between 1 and 12, + and may only be used in text mode. + Defaults to printing a calendar for the full year. + + +*Text-mode options:* + +.. option:: --width WIDTH, -w WIDTH + + The width of the date column in terminal columns. + The date is printed centred in the column. + Any value lower than 2 is ignored. + Defaults to 2. + + +.. option:: --lines LINES, -l LINES + + The number of lines for each week in terminal rows. + The date is printed top-aligned. + Any value lower than 1 is ignored. + Defaults to 1. + + +.. option:: --spacing SPACING, -s SPACING + + The space between months in columns. + Any value lower than 2 is ignored. + Defaults to 6. + + +.. option:: --months MONTHS, -m MONTHS + + The number of months printed per row. + Defaults to 3. + + +*HTML-mode options:* + +.. option:: --css CSS, -c CSS + + The path of a CSS stylesheet to use for the calendar. + This must either be relative to the generated HTML, + or an absolute HTTP or ``file:///`` URL. diff --git a/Lib/calendar.py b/Lib/calendar.py index e43ba4a078bcac..2a4deb70a0111f 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -721,7 +721,7 @@ def main(args=None): parser.add_argument( "-L", "--locale", default=None, - help="locale to be used from month and weekday names" + help="locale to use for month and weekday names" ) parser.add_argument( "-e", "--encoding", From 60a9eea3f56c002356998f5532b3ad870a1ffa8e Mon Sep 17 00:00:00 2001 From: Mikhail Samylov Date: Thu, 7 Sep 2023 00:29:46 +0300 Subject: [PATCH 532/598] Docs: Fix typo in datetime.tzinfo docstring (#107257) Co-authored-by: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Co-authored-by: Hugo van Kemenade --- Lib/_pydatetime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index df616bbaf8388d..4ffef65c577e1f 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1238,7 +1238,7 @@ def __reduce__(self): class tzinfo: """Abstract base class for time zone info classes. - Subclasses must override the name(), utcoffset() and dst() methods. + Subclasses must override the tzname(), utcoffset() and dst() methods. """ __slots__ = () From a52a3509770f29f940cda9307704908949912276 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Sep 2023 01:58:03 +0200 Subject: [PATCH 533/598] gh-109015: Add test.support.socket_helper.tcp_blackhole() (#109016) Skip test_asyncio, test_imaplib and test_socket tests if FreeBSD TCP blackhole is enabled (net.inet.tcp.blackhole=2). --- Lib/test/support/socket_helper.py | 60 +++++++++++++++++++ Lib/test/test_asyncio/test_events.py | 3 + Lib/test/test_asyncio/test_sock_lowlevel.py | 4 ++ Lib/test/test_asyncio/test_sslproto.py | 3 + Lib/test/test_imaplib.py | 1 + Lib/test/test_socket.py | 2 + ...-09-06-18-27-53.gh-issue-109015.1dS1AQ.rst | 6 ++ 7 files changed, 79 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-06-18-27-53.gh-issue-109015.1dS1AQ.rst diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 45f6d65c355dd4..87941ee1791b4e 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -3,6 +3,7 @@ import os.path import socket import sys +import subprocess import tempfile import unittest @@ -277,3 +278,62 @@ def create_unix_domain_name(): """ return tempfile.mktemp(prefix="test_python_", suffix='.sock', dir=os.path.curdir) + + +# consider that sysctl values should not change while tests are running +_sysctl_cache = {} + +def _get_sysctl(name): + """Get a sysctl value as an integer.""" + try: + return _sysctl_cache[name] + except KeyError: + pass + + # At least Linux and FreeBSD support the "-n" option + cmd = ['sysctl', '-n', name] + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + if proc.returncode: + support.print_warning(f'{' '.join(cmd)!r} command failed with ' + f'exit code {proc.returncode}') + # cache the error to only log the warning once + _sysctl_cache[name] = None + return None + output = proc.stdout + + # Parse '0\n' to get '0' + try: + value = int(output.strip()) + except Exception as exc: + support.print_warning(f'Failed to parse {' '.join(cmd)!r} ' + f'command output {output!r}: {exc!r}') + # cache the error to only log the warning once + _sysctl_cache[name] = None + return None + + _sysctl_cache[name] = value + return value + + +def tcp_blackhole(): + if not sys.platform.startswith('freebsd'): + return False + + # gh-109015: test if FreeBSD TCP blackhole is enabled + value = _get_sysctl('net.inet.tcp.blackhole') + if value is None: + # don't skip if we fail to get the sysctl value + return False + return (value != 0) + + +def skip_if_tcp_blackhole(test): + """Decorator skipping test if TCP blackhole is enabled.""" + skip_if = unittest.skipIf( + tcp_blackhole(), + "TCP blackhole is enabled (sysctl net.inet.tcp.blackhole)" + ) + return skip_if(test) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index b9069056c3a436..30cc8fd80bfe94 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -671,6 +671,7 @@ def test_create_connection_local_addr(self): self.assertEqual(port, expected) tr.close() + @socket_helper.skip_if_tcp_blackhole def test_create_connection_local_addr_skip_different_family(self): # See https://github.com/python/cpython/issues/86508 port1 = socket_helper.find_unused_port() @@ -692,6 +693,7 @@ async def getaddrinfo(host, port, *args, **kwargs): with self.assertRaises(OSError): self.loop.run_until_complete(f) + @socket_helper.skip_if_tcp_blackhole def test_create_connection_local_addr_nomatch_family(self): # See https://github.com/python/cpython/issues/86508 port1 = socket_helper.find_unused_port() @@ -1271,6 +1273,7 @@ def connection_made(self, transport): server.close() + @socket_helper.skip_if_tcp_blackhole def test_server_close(self): f = self.loop.create_server(MyProto, '0.0.0.0', 0) server = self.loop.run_until_complete(f) diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index b829fd4cc69fff..075113cbe8e4a6 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -10,6 +10,10 @@ from test import support from test.support import socket_helper +if socket_helper.tcp_blackhole(): + raise unittest.SkipTest('Not relevant to ProactorEventLoop') + + def tearDownModule(): asyncio.set_event_loop_policy(None) diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 52a45f1c7c6e96..37d015339761c6 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -5,6 +5,7 @@ import unittest import weakref from test import support +from test.support import socket_helper from unittest import mock try: import ssl @@ -350,6 +351,7 @@ async def client(addr): support.gc_collect() self.assertIsNone(client_context()) + @socket_helper.skip_if_tcp_blackhole def test_start_tls_client_buf_proto_1(self): HELLO_MSG = b'1' * self.PAYLOAD_SIZE @@ -502,6 +504,7 @@ async def client(addr): asyncio.wait_for(client(srv.addr), timeout=support.SHORT_TIMEOUT)) + @socket_helper.skip_if_tcp_blackhole def test_start_tls_server_1(self): HELLO_MSG = b'1' * self.PAYLOAD_SIZE ANSWER = b'answer' diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 89cdca499f92a9..a1eaf2169fe227 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -74,6 +74,7 @@ def test_that_Time2Internaldate_returns_a_result(self): for t in self.timevalues(): imaplib.Time2Internaldate(t) + @socket_helper.skip_if_tcp_blackhole def test_imap4_host_default_value(self): # Check whether the IMAP4_PORT is truly unavailable. with socket.socket() as s: diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 0eaf64257c3b81..f35618e0281e70 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5288,6 +5288,7 @@ def mocked_socket_module(self): finally: socket.socket = old_socket + @socket_helper.skip_if_tcp_blackhole def test_connect(self): port = socket_helper.find_unused_port() cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -5296,6 +5297,7 @@ def test_connect(self): cli.connect((HOST, port)) self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + @socket_helper.skip_if_tcp_blackhole def test_create_connection(self): # Issue #9792: errors raised by create_connection() should have # a proper errno attribute. diff --git a/Misc/NEWS.d/next/Tests/2023-09-06-18-27-53.gh-issue-109015.1dS1AQ.rst b/Misc/NEWS.d/next/Tests/2023-09-06-18-27-53.gh-issue-109015.1dS1AQ.rst new file mode 100644 index 00000000000000..cb641be9312e1a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-06-18-27-53.gh-issue-109015.1dS1AQ.rst @@ -0,0 +1,6 @@ +Fix test_asyncio, test_imaplib and test_socket tests on FreeBSD if the TCP +blackhole is enabled (``sysctl net.inet.tcp.blackhole``). Skip the few tests +which failed with ``ETIMEDOUT`` which such non standard configuration. +Currently, the `FreeBSD GCP image enables TCP and UDP blackhole +`_ (``sysctl net.inet.tcp.blackhole=2`` +and ``sysctl net.inet.udp.blackhole=1``). Patch by Victor Stinner. From 19eddb515a8cf87df7aaa347379b3e56c324385b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Sep 2023 02:09:06 +0200 Subject: [PATCH 534/598] gh-107211: No longer export internal _PyLong_FromUid() (#109037) No longer export _PyLong_FromUid() and _Py_Sigset_Converter() internal C API function. --- Modules/posixmodule.h | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h index 5452ffbf17acfc..8827ce153fed8c 100644 --- a/Modules/posixmodule.h +++ b/Modules/posixmodule.h @@ -2,31 +2,37 @@ #ifndef Py_POSIXMODULE_H #define Py_POSIXMODULE_H +#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif #ifdef HAVE_SYS_TYPES_H -#include +# include // uid_t #endif -#ifndef Py_LIMITED_API #ifndef MS_WINDOWS -PyAPI_FUNC(PyObject *) _PyLong_FromUid(uid_t); -PyAPI_FUNC(PyObject *) _PyLong_FromGid(gid_t); +extern PyObject* _PyLong_FromUid(uid_t); + +// Export for 'grp' shared extension +PyAPI_FUNC(PyObject*) _PyLong_FromGid(gid_t); + +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(int) _Py_Uid_Converter(PyObject *, uid_t *); + +// Export for 'grp' shared extension PyAPI_FUNC(int) _Py_Gid_Converter(PyObject *, gid_t *); -#endif /* MS_WINDOWS */ +#endif // !MS_WINDOWS -#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGWAIT) || \ - defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) -# define HAVE_SIGSET_T +#if (defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGWAIT) \ + || defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT)) +# define HAVE_SIGSET_T #endif -PyAPI_FUNC(int) _Py_Sigset_Converter(PyObject *, void *); -#endif /* Py_LIMITED_API */ +extern int _Py_Sigset_Converter(PyObject *, void *); #ifdef __cplusplus } #endif -#endif /* !Py_POSIXMODULE_H */ +#endif // !Py_LIMITED_API +#endif // !Py_POSIXMODULE_H From 3bfa24e29f286cbc1f42bdb4d2b1c0c9d643c8d6 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Thu, 7 Sep 2023 09:53:54 +0900 Subject: [PATCH 535/598] gh-107265: Remove all ENTER_EXECUTOR when execute _Py_Instrument (gh-108539) --- Include/internal/pycore_code.h | 3 +++ Lib/test/test_monitoring.py | 28 ++++++++++++++++++++++ Objects/codeobject.c | 17 ++++++++++++++ Python/instrumentation.c | 43 +++++++++------------------------- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 7c6629074758da..b3f480c7204172 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -233,6 +233,9 @@ extern void _PyLineTable_InitAddressRange( extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +/** API for executors */ +extern void _PyCode_Clear_Executors(PyCodeObject *code); + #define ENABLE_SPECIALIZATION 1 /* Specialization functions */ diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 95fe8d03b0b29a..8a7d300b734dcc 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1718,3 +1718,31 @@ def make_foo_optimized_then_set_event(): make_foo_optimized_then_set_event() finally: sys.monitoring.set_events(TEST_TOOL, 0) + + +class TestOptimizer(MonitoringTestBase, unittest.TestCase): + + def setUp(self): + import _testinternalcapi + self.old_opt = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.get_counter_optimizer() + _testinternalcapi.set_optimizer(opt) + super(TestOptimizer, self).setUp() + + def tearDown(self): + import _testinternalcapi + super(TestOptimizer, self).tearDown() + _testinternalcapi.set_optimizer(self.old_opt) + + def test_for_loop(self): + def test_func(x): + i = 0 + while i < x: + i += 1 + + code = test_func.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + test_func(1000) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 58306075cad48b..1443027ff293ac 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1479,6 +1479,23 @@ clear_executors(PyCodeObject *co) co->co_executors = NULL; } +void +_PyCode_Clear_Executors(PyCodeObject *code) { + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t opcode = instr->op.code; + uint8_t oparg = instr->op.arg; + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[oparg]; + assert(exec->vm_data.opcode != ENTER_EXECUTOR); + instr->op.code = exec->vm_data.opcode; + instr->op.arg = exec->vm_data.oparg; + } + } + clear_executors(code); +} + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9065043f55d8a7..acc3278d50a60a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -3,6 +3,7 @@ #include "opcode_ids.h" #include "pycore_call.h" +#include "pycore_code.h" // _PyCode_Clear_Executors() #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_long.h" @@ -583,13 +584,7 @@ de_instrument(PyCodeObject *code, int i, int event) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; - if (opcode == ENTER_EXECUTOR) { - int oparg = instr->op.arg; - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - opcode_ptr = &exec->vm_data.opcode; - opcode = *opcode_ptr; - assert(opcode != ENTER_EXECUTOR); - } + assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; @@ -734,22 +729,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); - #ifndef NDEBUG - _Py_CODEUNIT co_instr = _PyCode_CODE(code)[offset]; - uint8_t opcode = co_instr.op.code; - uint8_t oparg = co_instr.op.arg; - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; - opcode = exec->vm_data.oparg; - } - else { - opcode = _Py_GetBaseOpcode(code, offset); - } - assert(opcode != ENTER_EXECUTOR); - assert(opcode_has_event(opcode)); - #endif + assert(opcode_has_event(_Py_GetBaseOpcode(code, offset))); _PyCoMonitoringData *monitoring = code->_co_monitoring; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; @@ -1315,16 +1295,10 @@ initialize_tools(PyCodeObject *code) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; - int oparg = instr->op.arg; - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - opcode = exec->vm_data.opcode; - oparg = exec->vm_data.oparg; - } - else if (opcode == INSTRUMENTED_LINE) { + assert(opcode != ENTER_EXECUTOR); + if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[i].original_opcode; } - assert(opcode != ENTER_EXECUTOR); bool instrumented = is_instrumented(opcode); if (instrumented) { opcode = DE_INSTRUMENT[opcode]; @@ -1335,7 +1309,7 @@ initialize_tools(PyCodeObject *code) if (instrumented) { int8_t event; if (opcode == RESUME) { - event = oparg != 0; + event = instr->op.arg != 0; } else { event = EVENT_FOR_OPCODE[opcode]; @@ -1588,6 +1562,9 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) ); return 0; } + if (code->co_executors != NULL) { + _PyCode_Clear_Executors(code); + } int code_len = (int)Py_SIZE(code); /* code->_co_firsttraceable >= code_len indicates * that no instrumentation can be inserted. @@ -1629,7 +1606,9 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; CHECK(instr->op.code != 0); + assert(instr->op.code != ENTER_EXECUTOR); int base_opcode = _Py_GetBaseOpcode(code, i); + assert(base_opcode != ENTER_EXECUTOR); if (opcode_has_event(base_opcode)) { int8_t event; if (base_opcode == RESUME) { From f42edf1e7be5018a8988a219a168e231cbaa25e5 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 6 Sep 2023 19:42:58 -0700 Subject: [PATCH 536/598] gh-109045: Remove remaining LIMITED_API_AVAILABLE checks in tests (#109046) Commit 13a00078b81776b23b0b6add69b848382240d1f2 (#108663) made all Python builds compatible with the Limited API, and removed the LIMITED_API_AVAILABLE flag. However, some tests were still checking for that flag, so they were now being incorrectly skipped. Remove these checks to let these tests run again. Signed-off-by: Anders Kaseorg --- Lib/test/support/__init__.py | 3 +-- Modules/_testcapi/heaptype_relative.c | 4 ---- Modules/_testcapi/vectorcall_limited.c | 6 ------ Modules/_testcapimodule.c | 6 ------ 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d39f529499ca50..38ad965e155302 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1090,8 +1090,7 @@ def requires_limited_api(test): import _testcapi except ImportError: return unittest.skip('needs _testcapi module')(test) - return unittest.skipUnless( - _testcapi.LIMITED_API_AVAILABLE, 'needs Limited API support')(test) + return test def requires_specialization(test): return unittest.skipUnless( diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index c247ca33b33708..53dd01d1ed4f80 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -3,8 +3,6 @@ #include // max_align_t #include // memset -#ifdef LIMITED_API_AVAILABLE - static PyType_Slot empty_slots[] = { {0, NULL}, }; @@ -339,5 +337,3 @@ _PyTestCapi_Init_HeaptypeRelative(PyObject *m) { return 0; } - -#endif // LIMITED_API_AVAILABLE diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index a96925e840121a..e981c8625c2fb2 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,10 +1,6 @@ #define Py_LIMITED_API 0x030c0000 // 3.12 #include "parts.h" -#ifdef LIMITED_API_AVAILABLE - - - /* Test Vectorcall in the limited API */ static PyObject * @@ -175,5 +171,3 @@ _PyTestCapi_Init_VectorcallLimited(PyObject *m) { return 0; } - -#endif // LIMITED_API_AVAILABLE diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5ff5172f4f30eb..9bd963baf066a4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3996,18 +3996,12 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyAtomic(m) < 0) { return NULL; } - -#ifndef LIMITED_API_AVAILABLE - PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); -#else - PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_True); if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { return NULL; } if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { return NULL; } -#endif PyState_AddModule(m, &_testcapimodule); return m; From fd5989bda119f8c0f8571aad88d8e9be3be678a4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Sep 2023 04:47:57 +0200 Subject: [PATCH 537/598] gh-108753: _Py_PrintSpecializationStats() uses Py_hexdigits (#109040) --- Python/specialize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index e072167ff38ce9..8b4aac2f890930 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -290,8 +290,8 @@ _Py_PrintSpecializationStats(int to_file) char hex_name[41]; _PyOS_URandomNonblock(rand, 20); for (int i = 0; i < 20; i++) { - hex_name[2*i] = "0123456789abcdef"[rand[i]&15]; - hex_name[2*i+1] = "0123456789abcdef"[(rand[i]>>4)&15]; + hex_name[2*i] = Py_hexdigits[rand[i]&15]; + hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; } hex_name[40] = '\0'; char buf[64]; From e7d5433f944a5725aa82595f9251abfc8a63d333 Mon Sep 17 00:00:00 2001 From: Daniel Weiss <134341009+justdan6@users.noreply.github.com> Date: Wed, 6 Sep 2023 23:33:51 -0600 Subject: [PATCH 538/598] gh-108915: Removes extra backslashes in str.split docstring (#109044) --- Objects/clinic/unicodeobject.c.h | 6 +++--- Objects/unicodeobject.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index cfee9b8aa1fa8c..42b8a3c69cad8a 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -950,7 +950,7 @@ PyDoc_STRVAR(unicode_split__doc__, " The separator used to split the string.\n" "\n" " When set to None (the default value), will split on any whitespace\n" -" character (including \\\\n \\\\r \\\\t \\\\f and spaces) and will discard\n" +" character (including \\n \\r \\t \\f and spaces) and will discard\n" " empty strings from the result.\n" " maxsplit\n" " Maximum number of splits (starting from the left).\n" @@ -1074,7 +1074,7 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " The separator used to split the string.\n" "\n" " When set to None (the default value), will split on any whitespace\n" -" character (including \\\\n \\\\r \\\\t \\\\f and spaces) and will discard\n" +" character (including \\n \\r \\t \\f and spaces) and will discard\n" " empty strings from the result.\n" " maxsplit\n" " Maximum number of splits (starting from the left).\n" @@ -1504,4 +1504,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8d08dfbb814c4393 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4acdcfdc93f2a0f6 input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 045b48c526a958..4b87bf8e37aa94 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -12402,7 +12402,7 @@ str.split as unicode_split The separator used to split the string. When set to None (the default value), will split on any whitespace - character (including \\n \\r \\t \\f and spaces) and will discard + character (including \n \r \t \f and spaces) and will discard empty strings from the result. maxsplit: Py_ssize_t = -1 Maximum number of splits (starting from the left). @@ -12418,7 +12418,7 @@ the regular expression module. static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=906d953b44efc43b]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=07b9040d98c5fe8d]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); From babdced23fc299b7607ac76abfdd7a81050f8359 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Sep 2023 07:43:32 +0200 Subject: [PATCH 539/598] test.pythoninfo logs freedesktop_os_release() (#109057) --- Lib/test/pythoninfo.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 46522b50dd1e98..c628833478044e 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -164,6 +164,26 @@ def collect_platform(info_add): if libc_ver: info_add('platform.libc_ver', libc_ver) + try: + os_release = platform.freedesktop_os_release() + except OSError: + pass + else: + for key in ( + 'ID', + 'NAME', + 'PRETTY_NAME' + 'VARIANT', + 'VARIANT_ID', + 'VERSION', + 'VERSION_CODENAME', + 'VERSION_ID', + ): + if key not in os_release: + continue + info_add(f'platform.freedesktop_os_release[{key}]', + os_release[key]) + def collect_locale(info_add): import locale @@ -920,7 +940,6 @@ def dump_info(info, file=None): for key, value in infos: value = value.replace("\n", " ") print("%s: %s" % (key, value)) - print() def main(): @@ -929,6 +948,7 @@ def main(): dump_info(info) if error: + print() print("Collection failed: exit with error", file=sys.stderr) sys.exit(1) From 3e53ac99038920550358c1ea0212c3907a8cb385 Mon Sep 17 00:00:00 2001 From: Ijtaba Hussain Date: Thu, 7 Sep 2023 12:41:38 +0500 Subject: [PATCH 540/598] gh-103186: Suppress and assert expected RuntimeWarnings in test_sys_settrace (GH-103244) Caused as a result of frame manipulation where locals are never assigned / initialised. --- Lib/test/test_sys_settrace.py | 74 +++++++++++-------- ...-04-05-06-45-20.gh-issue-103186.640Eg-.rst | 1 + 2 files changed, 43 insertions(+), 32 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-04-05-06-45-20.gh-issue-103186.640Eg-.rst diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7d38addaee413e..53ec4eaea64b10 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -8,6 +8,7 @@ from functools import wraps import asyncio from test.support import import_helper +import contextlib support.requires_working_socket(module=True) @@ -1922,6 +1923,8 @@ def no_jump_without_trace_function(): class JumpTestCase(unittest.TestCase): + unbound_locals = r"assigning None to [0-9]+ unbound local" + def setUp(self): self.addCleanup(sys.settrace, sys.gettrace()) sys.settrace(None) @@ -1933,33 +1936,39 @@ def compare_jump_output(self, expected, received): "Received: " + repr(received)) def run_test(self, func, jumpFrom, jumpTo, expected, error=None, - event='line', decorated=False): + event='line', decorated=False, warning=None): tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] - if error is None: + + with contextlib.ExitStack() as stack: + if error is not None: + stack.enter_context(self.assertRaisesRegex(*error)) + if warning is not None: + stack.enter_context(self.assertWarnsRegex(*warning)) func(output) - else: - with self.assertRaisesRegex(*error): - func(output) + sys.settrace(None) self.compare_jump_output(expected, output) def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, - event='line', decorated=False): + event='line', decorated=False, warning=None): tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] - if error is None: + + with contextlib.ExitStack() as stack: + if error is not None: + stack.enter_context(self.assertRaisesRegex(*error)) + if warning is not None: + stack.enter_context(self.assertWarnsRegex(*warning)) asyncio.run(func(output)) - else: - with self.assertRaisesRegex(*error): - asyncio.run(func(output)) + sys.settrace(None) asyncio.set_event_loop_policy(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line', warning=None): """Decorator that creates a test that makes a jump from one place to another in the following code. """ @@ -1967,11 +1976,11 @@ def decorator(func): @wraps(func) def test(self): self.run_test(func, jumpFrom, jumpTo, expected, - error=error, event=event, decorated=True) + error=error, event=event, decorated=True, warning=warning) return test return decorator - def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line', warning=None): """Decorator that creates a test that makes a jump from one place to another in the following asynchronous code. """ @@ -1979,7 +1988,7 @@ def decorator(func): @wraps(func) def test(self): self.run_async_test(func, jumpFrom, jumpTo, expected, - error=error, event=event, decorated=True) + error=error, event=event, decorated=True, warning=warning) return test return decorator @@ -1996,7 +2005,7 @@ def test_jump_simple_backwards(output): output.append(1) output.append(2) - @jump_test(3, 5, [2, 5]) + @jump_test(3, 5, [2, 5], warning=(RuntimeWarning, unbound_locals)) def test_jump_out_of_block_forwards(output): for i in 1, 2: output.append(2) @@ -2210,7 +2219,7 @@ def test_jump_within_except_block(output): output.append(6) output.append(7) - @jump_test(6, 1, [1, 5, 1, 5]) + @jump_test(6, 1, [1, 5, 1, 5], warning=(RuntimeWarning, unbound_locals)) def test_jump_over_try_except(output): output.append(1) try: @@ -2306,7 +2315,7 @@ def test_jump_out_of_complex_nested_blocks(output): output.append(11) output.append(12) - @jump_test(3, 5, [1, 2, 5]) + @jump_test(3, 5, [1, 2, 5], warning=(RuntimeWarning, unbound_locals)) def test_jump_out_of_with_assignment(output): output.append(1) with tracecontext(output, 2) \ @@ -2314,7 +2323,7 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) - @async_jump_test(3, 5, [1, 2, 5]) + @async_jump_test(3, 5, [1, 2, 5], warning=(RuntimeWarning, unbound_locals)) async def test_jump_out_of_async_with_assignment(output): output.append(1) async with asynctracecontext(output, 2) \ @@ -2350,7 +2359,7 @@ def test_jump_over_break_in_try_finally_block(output): break output.append(13) - @jump_test(1, 7, [7, 8]) + @jump_test(1, 7, [7, 8], warning=(RuntimeWarning, unbound_locals)) def test_jump_over_for_block_before_else(output): output.append(1) if not output: # always false @@ -2361,7 +2370,7 @@ def test_jump_over_for_block_before_else(output): output.append(7) output.append(8) - @async_jump_test(1, 7, [7, 8]) + @async_jump_test(1, 7, [7, 8], warning=(RuntimeWarning, unbound_locals)) async def test_jump_over_async_for_block_before_else(output): output.append(1) if not output: # always false @@ -2436,6 +2445,7 @@ def test_no_jump_backwards_into_for_block(output): output.append(2) output.append(3) + @async_jump_test(3, 2, [2, 2], (ValueError, "can't jump into the body of a for loop")) async def test_no_jump_backwards_into_async_for_block(output): async for i in asynciter([1, 2]): @@ -2501,7 +2511,7 @@ def test_jump_backwards_into_try_except_block(output): output.append(6) # 'except' with a variable creates an implicit finally block - @jump_test(5, 7, [4, 7, 8]) + @jump_test(5, 7, [4, 7, 8], warning=(RuntimeWarning, unbound_locals)) def test_jump_between_except_blocks_2(output): try: 1/0 @@ -2664,7 +2674,7 @@ def test_large_function(self): output.append(x) # line 1007 return""" % ('\n' * 1000,), d) f = d['f'] - self.run_test(f, 2, 1007, [0]) + self.run_test(f, 2, 1007, [0], warning=(RuntimeWarning, self.unbound_locals)) def test_jump_to_firstlineno(self): # This tests that PDB can jump back to the first line in a @@ -2714,7 +2724,7 @@ def gen(): next(gen()) output.append(5) - @jump_test(2, 3, [1, 3]) + @jump_test(2, 3, [1, 3], warning=(RuntimeWarning, unbound_locals)) def test_jump_forward_over_listcomp(output): output.append(1) x = [i for i in range(10)] @@ -2722,13 +2732,13 @@ def test_jump_forward_over_listcomp(output): # checking for segfaults. # See https://github.com/python/cpython/issues/92311 - @jump_test(3, 1, []) + @jump_test(3, 1, [], warning=(RuntimeWarning, unbound_locals)) def test_jump_backward_over_listcomp(output): a = 1 x = [i for i in range(10)] c = 3 - @jump_test(8, 2, [2, 7, 2]) + @jump_test(8, 2, [2, 7, 2], warning=(RuntimeWarning, unbound_locals)) def test_jump_backward_over_listcomp_v2(output): flag = False output.append(2) @@ -2739,19 +2749,19 @@ def test_jump_backward_over_listcomp_v2(output): output.append(7) output.append(8) - @async_jump_test(2, 3, [1, 3]) + @async_jump_test(2, 3, [1, 3], warning=(RuntimeWarning, unbound_locals)) async def test_jump_forward_over_async_listcomp(output): output.append(1) x = [i async for i in asynciter(range(10))] output.append(3) - @async_jump_test(3, 1, []) + @async_jump_test(3, 1, [], warning=(RuntimeWarning, unbound_locals)) async def test_jump_backward_over_async_listcomp(output): a = 1 x = [i async for i in asynciter(range(10))] c = 3 - @async_jump_test(8, 2, [2, 7, 2]) + @async_jump_test(8, 2, [2, 7, 2], warning=(RuntimeWarning, unbound_locals)) async def test_jump_backward_over_async_listcomp_v2(output): flag = False output.append(2) @@ -2820,13 +2830,13 @@ def test_jump_with_null_on_stack_load_attr(output): ) output.append(15) - @jump_test(2, 3, [1, 3]) + @jump_test(2, 3, [1, 3], warning=(RuntimeWarning, unbound_locals)) def test_jump_extended_args_unpack_ex_simple(output): output.append(1) _, *_, _ = output.append(2) or "Spam" output.append(3) - @jump_test(3, 4, [1, 4, 4, 5]) + @jump_test(3, 4, [1, 4, 4, 5], warning=(RuntimeWarning, unbound_locals)) def test_jump_extended_args_unpack_ex_tricky(output): output.append(1) ( @@ -2848,9 +2858,9 @@ def test_jump_extended_args_for_iter(self): namespace = {} exec("\n".join(source), namespace) f = namespace["f"] - self.run_test(f, 2, 100_000, [1, 100_000]) + self.run_test(f, 2, 100_000, [1, 100_000], warning=(RuntimeWarning, self.unbound_locals)) - @jump_test(2, 3, [1, 3]) + @jump_test(2, 3, [1, 3], warning=(RuntimeWarning, unbound_locals)) def test_jump_or_pop(output): output.append(1) _ = output.append(2) and "Spam" diff --git a/Misc/NEWS.d/next/Tests/2023-04-05-06-45-20.gh-issue-103186.640Eg-.rst b/Misc/NEWS.d/next/Tests/2023-04-05-06-45-20.gh-issue-103186.640Eg-.rst new file mode 100644 index 00000000000000..2f596aa5f47bda --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-04-05-06-45-20.gh-issue-103186.640Eg-.rst @@ -0,0 +1 @@ +Suppress and assert expected RuntimeWarnings in test_sys_settrace.py From e183a71eef1ec3ac86bb4d81a158c21d6f1a783b Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Thu, 7 Sep 2023 14:49:13 +0530 Subject: [PATCH 541/598] bpo-38157: Add example about per file output for mock_open. (#16090) Co-authored-by: Stanley <46876382+slateny@users.noreply.github.com> Co-authored-by: Jelle Zijlstra Co-authored-by: Hugo van Kemenade --- Doc/library/unittest.mock-examples.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 744fc9de63cd16..34f343ebacdbb7 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -360,6 +360,30 @@ of arbitrary attributes as well as the getting of them then you can use *spec_set* instead of *spec*. +Using side_effect to return per file content +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:func:`mock_open` is used to patch :func:`open` method. :attr:`~Mock.side_effect` +can be used to return a new Mock object per call. This can be used to return different +contents per file stored in a dictionary:: + + DEFAULT = "default" + data_dict = {"file1": "data1", + "file2": "data2"} + + def open_side_effect(name): + return mock_open(read_data=data_dict.get(name, DEFAULT))() + + with patch("builtins.open", side_effect=open_side_effect): + with open("file1") as file1: + assert file1.read() == "data1" + + with open("file2") as file2: + assert file2.read() == "data2" + + with open("file3") as file2: + assert file2.read() == "default" + Patch Decorators ---------------- From 6b15ff52351787644115a4dd9d5d6717d66b9806 Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Thu, 7 Sep 2023 13:33:02 +0300 Subject: [PATCH 542/598] socket documentation fix - rename triple to 3-tuple (#24722) Co-authored-by: Hugo van Kemenade --- Doc/library/socket.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 4f220e8a098979..83957c87990440 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -979,7 +979,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: gethostbyname_ex(hostname) Translate a host name to IPv4 address format, extended interface. Return a - triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's + 3-tuple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's primary host name, *aliaslist* is a (possibly empty) list of alternative host names for the same address, and *ipaddrlist* is a list of IPv4 addresses for the same interface on the same host (often but not @@ -1007,7 +1007,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: gethostbyaddr(ip_address) - Return a triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the + Return a 3-tuple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the primary host name responding to the given *ip_address*, *aliaslist* is a (possibly empty) list of alternative host names for the same address, and *ipaddrlist* is a list of IPv4/v6 addresses for the same interface on the same From 1294fcede09af6c781553b7a3a6ff612c7dfa431 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Thu, 7 Sep 2023 14:14:27 +0100 Subject: [PATCH 543/598] GH-90915: Document that SystemExit doesn't trigger sys.excepthook (#31357) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Hugo van Kemenade --- Doc/library/sys.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ba8d80bccbd894..c116f4b9b00825 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -378,7 +378,7 @@ always available. This function prints out a given traceback and exception to ``sys.stderr``. - When an exception is raised and uncaught, the interpreter calls + When an exception other than :exc:`SystemExit` is raised and uncaught, the interpreter calls ``sys.excepthook`` with three arguments, the exception class, exception instance, and a traceback object. In an interactive session this happens just before control is returned to the prompt; in a Python program this happens just From e4bb0026b9a21d066e7a5c4716ea4d755b95d2d5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 16:30:41 +0300 Subject: [PATCH 544/598] gh-103186: Remove debug print in test_sys_settrace (GH-109077) --- Lib/test/test_sys_settrace.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 53ec4eaea64b10..2888f0b870672d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1654,7 +1654,6 @@ def error_once(frame, event, arg): except Exception as ex: count = 0 tb = ex.__traceback__ - print(tb) while tb: if tb.tb_frame.f_code.co_name == "test_settrace_error": count += 1 From d485551c9d1792ff3539eef1d6374bd4c01dcd5d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 16:34:44 +0300 Subject: [PATCH 545/598] gh-103186: Suppress RuntimeWarning about unclosed async iterator in test_sys_settrace (GH-109075) --- Lib/test/test_sys_settrace.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 2888f0b870672d..232900876392ff 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -41,6 +41,20 @@ async def asynciter(iterable): for x in iterable: yield x +def clean_asynciter(test): + @wraps(test) + async def wrapper(*args, **kwargs): + cleanups = [] + def wrapped_asynciter(iterable): + it = asynciter(iterable) + cleanups.append(it.aclose) + return it + try: + return await test(*args, **kwargs, asynciter=wrapped_asynciter) + finally: + while cleanups: + await cleanups.pop()() + return wrapper # A very basic example. If this fails, we're in deep trouble. def basic(): @@ -1936,7 +1950,11 @@ def compare_jump_output(self, expected, received): def run_test(self, func, jumpFrom, jumpTo, expected, error=None, event='line', decorated=False, warning=None): - tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + wrapped = func + while hasattr(wrapped, '__wrapped__'): + wrapped = wrapped.__wrapped__ + + tracer = JumpTracer(wrapped, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] @@ -1952,7 +1970,11 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, event='line', decorated=False, warning=None): - tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + wrapped = func + while hasattr(wrapped, '__wrapped__'): + wrapped = wrapped.__wrapped__ + + tracer = JumpTracer(wrapped, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] @@ -2023,7 +2045,8 @@ def test_jump_out_of_block_backwards(output): output.append(7) @async_jump_test(4, 5, [3, 5]) - async def test_jump_out_of_async_for_block_forwards(output): + @clean_asynciter + async def test_jump_out_of_async_for_block_forwards(output, asynciter): for i in [1]: async for i in asynciter([1, 2]): output.append(3) @@ -2031,7 +2054,8 @@ async def test_jump_out_of_async_for_block_forwards(output): output.append(5) @async_jump_test(5, 2, [2, 4, 2, 4, 5, 6]) - async def test_jump_out_of_async_for_block_backwards(output): + @clean_asynciter + async def test_jump_out_of_async_for_block_backwards(output, asynciter): for i in [1]: output.append(2) async for i in asynciter([1]): From 0858328ca2457ae95715eb93e347d5c0547bec6f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 7 Sep 2023 14:39:03 +0100 Subject: [PATCH 546/598] GH-108614: Add `RESUME_CHECK` instruction (GH-108630) --- Include/internal/pycore_opcode_metadata.h | 10 +- Include/opcode_ids.h | 283 ++++++++--------- Lib/_opcode_metadata.py | 286 ++++++++--------- Lib/test/test_dis.py | 290 +++++++++--------- ...-08-26-10-36-45.gh-issue-108614.wl5l-W.rst | 2 + Objects/genobject.c | 6 +- Programs/test_frozenmain.h | 22 +- Python/abstract_interp_cases.c.h | 2 +- Python/bytecodes.c | 29 +- Python/ceval_macros.h | 2 + Python/emscripten_signal.c | 8 +- Python/executor_cases.c.h | 25 +- Python/generated_cases.c.h | 28 +- Python/opcode_targets.h | 4 +- Tools/cases_generator/instructions.py | 1 + Tools/cases_generator/parsing.py | 3 +- 16 files changed, 519 insertions(+), 482 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-26-10-36-45.gh-issue-108614.wl5l-W.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fa4cbd9ed31c01..fb5c0465a10021 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -82,6 +82,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case RESUME: return 0; + case RESUME_CHECK: + return 0; case INSTRUMENTED_RESUME: return 0; case LOAD_CLOSURE: @@ -614,6 +616,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RESUME: return 0; + case RESUME_CHECK: + return 0; case INSTRUMENTED_RESUME: return 0; case LOAD_CLOSURE: @@ -1207,6 +1211,7 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SI const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, @@ -1477,7 +1482,7 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACR #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, - [RESUME] = { .nuops = 1, .uops = { { RESUME, 0, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { RESUME_CHECK, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, @@ -1716,6 +1721,7 @@ const char *const _PyOpcode_OpName[268] = { [POP_TOP] = "POP_TOP", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [PUSH_NULL] = "PUSH_NULL", + [RESUME_CHECK] = "RESUME_CHECK", [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", @@ -2077,6 +2083,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [RERAISE] = RERAISE, [RESERVED] = RESERVED, [RESUME] = RESUME, + [RESUME_CHECK] = RESUME, [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, @@ -2122,7 +2129,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 188: \ case 189: \ case 190: \ case 191: \ diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index cd43716415d1db..eabdf4bc020ef7 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -57,148 +57,149 @@ extern "C" { #define POP_TOP 44 #define PUSH_EXC_INFO 45 #define PUSH_NULL 46 -#define RETURN_GENERATOR 47 -#define RETURN_VALUE 48 -#define SETUP_ANNOTATIONS 49 -#define STORE_ATTR_INSTANCE_VALUE 50 -#define STORE_ATTR_SLOT 51 -#define STORE_SLICE 52 -#define STORE_SUBSCR 53 -#define STORE_SUBSCR_DICT 54 -#define STORE_SUBSCR_LIST_INT 55 -#define TO_BOOL 56 -#define TO_BOOL_ALWAYS_TRUE 57 -#define TO_BOOL_BOOL 58 -#define TO_BOOL_INT 59 -#define TO_BOOL_LIST 60 -#define TO_BOOL_NONE 61 -#define TO_BOOL_STR 62 -#define UNARY_INVERT 63 -#define UNARY_NEGATIVE 64 -#define UNARY_NOT 65 -#define WITH_EXCEPT_START 66 -#define HAVE_ARGUMENT 67 -#define BINARY_OP 67 -#define BUILD_CONST_KEY_MAP 68 -#define BUILD_LIST 69 -#define BUILD_MAP 70 -#define BUILD_SET 71 -#define BUILD_SLICE 72 -#define BUILD_STRING 73 -#define BUILD_TUPLE 74 -#define CALL 75 -#define CALL_BOUND_METHOD_EXACT_ARGS 76 -#define CALL_BUILTIN_CLASS 77 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 78 -#define CALL_FUNCTION_EX 79 -#define CALL_INTRINSIC_1 80 -#define CALL_INTRINSIC_2 81 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 82 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 83 -#define CALL_NO_KW_BUILTIN_FAST 84 -#define CALL_NO_KW_BUILTIN_O 85 -#define CALL_NO_KW_ISINSTANCE 86 -#define CALL_NO_KW_LEN 87 -#define CALL_NO_KW_LIST_APPEND 88 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 89 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 90 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 91 -#define CALL_NO_KW_STR_1 92 -#define CALL_NO_KW_TUPLE_1 93 -#define CALL_NO_KW_TYPE_1 94 -#define CALL_PY_EXACT_ARGS 95 -#define CALL_PY_WITH_DEFAULTS 96 -#define COMPARE_OP 97 -#define COMPARE_OP_FLOAT 98 -#define COMPARE_OP_INT 99 -#define COMPARE_OP_STR 100 -#define CONTAINS_OP 101 -#define CONVERT_VALUE 102 -#define COPY 103 -#define COPY_FREE_VARS 104 -#define DELETE_ATTR 105 -#define DELETE_DEREF 106 -#define DELETE_FAST 107 -#define DELETE_GLOBAL 108 -#define DELETE_NAME 109 -#define DICT_MERGE 110 -#define DICT_UPDATE 111 -#define ENTER_EXECUTOR 112 -#define EXTENDED_ARG 113 -#define FOR_ITER 114 -#define FOR_ITER_GEN 115 -#define FOR_ITER_LIST 116 -#define FOR_ITER_RANGE 117 -#define FOR_ITER_TUPLE 118 -#define GET_AWAITABLE 119 -#define IMPORT_FROM 120 -#define IMPORT_NAME 121 -#define IS_OP 122 -#define JUMP_BACKWARD 123 -#define JUMP_BACKWARD_NO_INTERRUPT 124 -#define JUMP_FORWARD 125 -#define KW_NAMES 126 -#define LIST_APPEND 127 -#define LIST_EXTEND 128 -#define LOAD_ATTR 129 -#define LOAD_ATTR_CLASS 130 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 131 -#define LOAD_ATTR_INSTANCE_VALUE 132 -#define LOAD_ATTR_METHOD_LAZY_DICT 133 -#define LOAD_ATTR_METHOD_NO_DICT 134 -#define LOAD_ATTR_METHOD_WITH_VALUES 135 -#define LOAD_ATTR_MODULE 136 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 137 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 138 -#define LOAD_ATTR_PROPERTY 139 -#define LOAD_ATTR_SLOT 140 -#define LOAD_ATTR_WITH_HINT 141 -#define LOAD_CONST 142 -#define LOAD_DEREF 143 -#define LOAD_FAST 144 -#define LOAD_FAST_AND_CLEAR 145 -#define LOAD_FAST_CHECK 146 -#define LOAD_FAST_LOAD_FAST 147 -#define LOAD_FROM_DICT_OR_DEREF 148 -#define LOAD_FROM_DICT_OR_GLOBALS 149 -#define LOAD_GLOBAL 150 -#define LOAD_GLOBAL_BUILTIN 151 -#define LOAD_GLOBAL_MODULE 152 -#define LOAD_NAME 153 -#define LOAD_SUPER_ATTR 154 -#define LOAD_SUPER_ATTR_ATTR 155 -#define LOAD_SUPER_ATTR_METHOD 156 -#define MAKE_CELL 157 -#define MAP_ADD 158 -#define MATCH_CLASS 159 -#define POP_JUMP_IF_FALSE 160 -#define POP_JUMP_IF_NONE 161 -#define POP_JUMP_IF_NOT_NONE 162 -#define POP_JUMP_IF_TRUE 163 -#define RAISE_VARARGS 164 -#define RERAISE 165 +#define RESUME_CHECK 47 +#define RETURN_GENERATOR 48 +#define RETURN_VALUE 49 +#define SETUP_ANNOTATIONS 50 +#define STORE_ATTR_INSTANCE_VALUE 51 +#define STORE_ATTR_SLOT 52 +#define STORE_SLICE 53 +#define STORE_SUBSCR 54 +#define STORE_SUBSCR_DICT 55 +#define STORE_SUBSCR_LIST_INT 56 +#define TO_BOOL 57 +#define TO_BOOL_ALWAYS_TRUE 58 +#define TO_BOOL_BOOL 59 +#define TO_BOOL_INT 60 +#define TO_BOOL_LIST 61 +#define TO_BOOL_NONE 62 +#define TO_BOOL_STR 63 +#define UNARY_INVERT 64 +#define UNARY_NEGATIVE 65 +#define UNARY_NOT 66 +#define WITH_EXCEPT_START 67 +#define HAVE_ARGUMENT 68 +#define BINARY_OP 68 +#define BUILD_CONST_KEY_MAP 69 +#define BUILD_LIST 70 +#define BUILD_MAP 71 +#define BUILD_SET 72 +#define BUILD_SLICE 73 +#define BUILD_STRING 74 +#define BUILD_TUPLE 75 +#define CALL 76 +#define CALL_BOUND_METHOD_EXACT_ARGS 77 +#define CALL_BUILTIN_CLASS 78 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 79 +#define CALL_FUNCTION_EX 80 +#define CALL_INTRINSIC_1 81 +#define CALL_INTRINSIC_2 82 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 83 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 84 +#define CALL_NO_KW_BUILTIN_FAST 85 +#define CALL_NO_KW_BUILTIN_O 86 +#define CALL_NO_KW_ISINSTANCE 87 +#define CALL_NO_KW_LEN 88 +#define CALL_NO_KW_LIST_APPEND 89 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 90 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 91 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 92 +#define CALL_NO_KW_STR_1 93 +#define CALL_NO_KW_TUPLE_1 94 +#define CALL_NO_KW_TYPE_1 95 +#define CALL_PY_EXACT_ARGS 96 +#define CALL_PY_WITH_DEFAULTS 97 +#define COMPARE_OP 98 +#define COMPARE_OP_FLOAT 99 +#define COMPARE_OP_INT 100 +#define COMPARE_OP_STR 101 +#define CONTAINS_OP 102 +#define CONVERT_VALUE 103 +#define COPY 104 +#define COPY_FREE_VARS 105 +#define DELETE_ATTR 106 +#define DELETE_DEREF 107 +#define DELETE_FAST 108 +#define DELETE_GLOBAL 109 +#define DELETE_NAME 110 +#define DICT_MERGE 111 +#define DICT_UPDATE 112 +#define ENTER_EXECUTOR 113 +#define EXTENDED_ARG 114 +#define FOR_ITER 115 +#define FOR_ITER_GEN 116 +#define FOR_ITER_LIST 117 +#define FOR_ITER_RANGE 118 +#define FOR_ITER_TUPLE 119 +#define GET_AWAITABLE 120 +#define IMPORT_FROM 121 +#define IMPORT_NAME 122 +#define IS_OP 123 +#define JUMP_BACKWARD 124 +#define JUMP_BACKWARD_NO_INTERRUPT 125 +#define JUMP_FORWARD 126 +#define KW_NAMES 127 +#define LIST_APPEND 128 +#define LIST_EXTEND 129 +#define LOAD_ATTR 130 +#define LOAD_ATTR_CLASS 131 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 132 +#define LOAD_ATTR_INSTANCE_VALUE 133 +#define LOAD_ATTR_METHOD_LAZY_DICT 134 +#define LOAD_ATTR_METHOD_NO_DICT 135 +#define LOAD_ATTR_METHOD_WITH_VALUES 136 +#define LOAD_ATTR_MODULE 137 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 138 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 139 +#define LOAD_ATTR_PROPERTY 140 +#define LOAD_ATTR_SLOT 141 +#define LOAD_ATTR_WITH_HINT 142 +#define LOAD_CONST 143 +#define LOAD_DEREF 144 +#define LOAD_FAST 145 +#define LOAD_FAST_AND_CLEAR 146 +#define LOAD_FAST_CHECK 147 +#define LOAD_FAST_LOAD_FAST 148 +#define LOAD_FROM_DICT_OR_DEREF 149 +#define LOAD_FROM_DICT_OR_GLOBALS 150 +#define LOAD_GLOBAL 151 +#define LOAD_GLOBAL_BUILTIN 152 +#define LOAD_GLOBAL_MODULE 153 +#define LOAD_NAME 154 +#define LOAD_SUPER_ATTR 155 +#define LOAD_SUPER_ATTR_ATTR 156 +#define LOAD_SUPER_ATTR_METHOD 157 +#define MAKE_CELL 158 +#define MAP_ADD 159 +#define MATCH_CLASS 160 +#define POP_JUMP_IF_FALSE 161 +#define POP_JUMP_IF_NONE 162 +#define POP_JUMP_IF_NOT_NONE 163 +#define POP_JUMP_IF_TRUE 164 +#define RAISE_VARARGS 165 #define RESUME 166 -#define RETURN_CONST 167 -#define SEND 168 -#define SEND_GEN 169 -#define SET_ADD 170 -#define SET_FUNCTION_ATTRIBUTE 171 -#define SET_UPDATE 172 -#define STORE_ATTR 173 -#define STORE_ATTR_WITH_HINT 174 -#define STORE_DEREF 175 -#define STORE_FAST 176 -#define STORE_FAST_LOAD_FAST 177 -#define STORE_FAST_STORE_FAST 178 -#define STORE_GLOBAL 179 -#define STORE_NAME 180 -#define SWAP 181 -#define UNPACK_EX 182 -#define UNPACK_SEQUENCE 183 -#define UNPACK_SEQUENCE_LIST 184 -#define UNPACK_SEQUENCE_TUPLE 185 -#define UNPACK_SEQUENCE_TWO_TUPLE 186 -#define YIELD_VALUE 187 +#define RERAISE 167 +#define RETURN_CONST 168 +#define SEND 169 +#define SEND_GEN 170 +#define SET_ADD 171 +#define SET_FUNCTION_ATTRIBUTE 172 +#define SET_UPDATE 173 +#define STORE_ATTR 174 +#define STORE_ATTR_WITH_HINT 175 +#define STORE_DEREF 176 +#define STORE_FAST 177 +#define STORE_FAST_LOAD_FAST 178 +#define STORE_FAST_STORE_FAST 179 +#define STORE_GLOBAL 180 +#define STORE_NAME 181 +#define SWAP 182 +#define UNPACK_EX 183 +#define UNPACK_SEQUENCE 184 +#define UNPACK_SEQUENCE_LIST 185 +#define UNPACK_SEQUENCE_TUPLE 186 +#define UNPACK_SEQUENCE_TWO_TUPLE 187 +#define YIELD_VALUE 188 #define MIN_INSTRUMENTED_OPCODE 237 #define INSTRUMENTED_RESUME 237 #define INSTRUMENTED_END_FOR 238 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b02aa771c347e7..20975ffb4c5321 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -4,6 +4,9 @@ # Do not edit! _specializations = { + "RESUME": [ + "RESUME_CHECK", + ], "TO_BOOL": [ "TO_BOOL_ALWAYS_TRUE", "TO_BOOL_BOOL", @@ -117,62 +120,63 @@ 'BINARY_SUBSCR_LIST_INT': 15, 'BINARY_SUBSCR_STR_INT': 16, 'BINARY_SUBSCR_TUPLE_INT': 18, - 'STORE_ATTR_INSTANCE_VALUE': 50, - 'STORE_ATTR_SLOT': 51, - 'STORE_SUBSCR_DICT': 54, - 'STORE_SUBSCR_LIST_INT': 55, - 'TO_BOOL_ALWAYS_TRUE': 57, - 'TO_BOOL_BOOL': 58, - 'TO_BOOL_INT': 59, - 'TO_BOOL_LIST': 60, - 'TO_BOOL_NONE': 61, - 'TO_BOOL_STR': 62, - 'CALL_BOUND_METHOD_EXACT_ARGS': 76, - 'CALL_BUILTIN_CLASS': 77, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 78, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 82, - 'CALL_NO_KW_ALLOC_AND_ENTER_INIT': 83, - 'CALL_NO_KW_BUILTIN_FAST': 84, - 'CALL_NO_KW_BUILTIN_O': 85, - 'CALL_NO_KW_ISINSTANCE': 86, - 'CALL_NO_KW_LEN': 87, - 'CALL_NO_KW_LIST_APPEND': 88, - 'CALL_NO_KW_METHOD_DESCRIPTOR_FAST': 89, - 'CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS': 90, - 'CALL_NO_KW_METHOD_DESCRIPTOR_O': 91, - 'CALL_NO_KW_STR_1': 92, - 'CALL_NO_KW_TUPLE_1': 93, - 'CALL_NO_KW_TYPE_1': 94, - 'CALL_PY_EXACT_ARGS': 95, - 'CALL_PY_WITH_DEFAULTS': 96, - 'COMPARE_OP_FLOAT': 98, - 'COMPARE_OP_INT': 99, - 'COMPARE_OP_STR': 100, - 'FOR_ITER_GEN': 115, - 'FOR_ITER_LIST': 116, - 'FOR_ITER_RANGE': 117, - 'FOR_ITER_TUPLE': 118, - 'LOAD_ATTR_CLASS': 130, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 131, - 'LOAD_ATTR_INSTANCE_VALUE': 132, - 'LOAD_ATTR_METHOD_LAZY_DICT': 133, - 'LOAD_ATTR_METHOD_NO_DICT': 134, - 'LOAD_ATTR_METHOD_WITH_VALUES': 135, - 'LOAD_ATTR_MODULE': 136, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 137, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 138, - 'LOAD_ATTR_PROPERTY': 139, - 'LOAD_ATTR_SLOT': 140, - 'LOAD_ATTR_WITH_HINT': 141, - 'LOAD_GLOBAL_BUILTIN': 151, - 'LOAD_GLOBAL_MODULE': 152, - 'LOAD_SUPER_ATTR_ATTR': 155, - 'LOAD_SUPER_ATTR_METHOD': 156, - 'SEND_GEN': 169, - 'STORE_ATTR_WITH_HINT': 174, - 'UNPACK_SEQUENCE_LIST': 184, - 'UNPACK_SEQUENCE_TUPLE': 185, - 'UNPACK_SEQUENCE_TWO_TUPLE': 186, + 'RESUME_CHECK': 47, + 'STORE_ATTR_INSTANCE_VALUE': 51, + 'STORE_ATTR_SLOT': 52, + 'STORE_SUBSCR_DICT': 55, + 'STORE_SUBSCR_LIST_INT': 56, + 'TO_BOOL_ALWAYS_TRUE': 58, + 'TO_BOOL_BOOL': 59, + 'TO_BOOL_INT': 60, + 'TO_BOOL_LIST': 61, + 'TO_BOOL_NONE': 62, + 'TO_BOOL_STR': 63, + 'CALL_BOUND_METHOD_EXACT_ARGS': 77, + 'CALL_BUILTIN_CLASS': 78, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 79, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 83, + 'CALL_NO_KW_ALLOC_AND_ENTER_INIT': 84, + 'CALL_NO_KW_BUILTIN_FAST': 85, + 'CALL_NO_KW_BUILTIN_O': 86, + 'CALL_NO_KW_ISINSTANCE': 87, + 'CALL_NO_KW_LEN': 88, + 'CALL_NO_KW_LIST_APPEND': 89, + 'CALL_NO_KW_METHOD_DESCRIPTOR_FAST': 90, + 'CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS': 91, + 'CALL_NO_KW_METHOD_DESCRIPTOR_O': 92, + 'CALL_NO_KW_STR_1': 93, + 'CALL_NO_KW_TUPLE_1': 94, + 'CALL_NO_KW_TYPE_1': 95, + 'CALL_PY_EXACT_ARGS': 96, + 'CALL_PY_WITH_DEFAULTS': 97, + 'COMPARE_OP_FLOAT': 99, + 'COMPARE_OP_INT': 100, + 'COMPARE_OP_STR': 101, + 'FOR_ITER_GEN': 116, + 'FOR_ITER_LIST': 117, + 'FOR_ITER_RANGE': 118, + 'FOR_ITER_TUPLE': 119, + 'LOAD_ATTR_CLASS': 131, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 132, + 'LOAD_ATTR_INSTANCE_VALUE': 133, + 'LOAD_ATTR_METHOD_LAZY_DICT': 134, + 'LOAD_ATTR_METHOD_NO_DICT': 135, + 'LOAD_ATTR_METHOD_WITH_VALUES': 136, + 'LOAD_ATTR_MODULE': 137, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 138, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 139, + 'LOAD_ATTR_PROPERTY': 140, + 'LOAD_ATTR_SLOT': 141, + 'LOAD_ATTR_WITH_HINT': 142, + 'LOAD_GLOBAL_BUILTIN': 152, + 'LOAD_GLOBAL_MODULE': 153, + 'LOAD_SUPER_ATTR_ATTR': 156, + 'LOAD_SUPER_ATTR_METHOD': 157, + 'SEND_GEN': 170, + 'STORE_ATTR_WITH_HINT': 175, + 'UNPACK_SEQUENCE_LIST': 185, + 'UNPACK_SEQUENCE_TUPLE': 186, + 'UNPACK_SEQUENCE_TWO_TUPLE': 187, } opmap = { @@ -210,91 +214,91 @@ 'POP_TOP': 44, 'PUSH_EXC_INFO': 45, 'PUSH_NULL': 46, - 'RETURN_GENERATOR': 47, - 'RETURN_VALUE': 48, - 'SETUP_ANNOTATIONS': 49, - 'STORE_SLICE': 52, - 'STORE_SUBSCR': 53, - 'TO_BOOL': 56, - 'UNARY_INVERT': 63, - 'UNARY_NEGATIVE': 64, - 'UNARY_NOT': 65, - 'WITH_EXCEPT_START': 66, - 'BINARY_OP': 67, - 'BUILD_CONST_KEY_MAP': 68, - 'BUILD_LIST': 69, - 'BUILD_MAP': 70, - 'BUILD_SET': 71, - 'BUILD_SLICE': 72, - 'BUILD_STRING': 73, - 'BUILD_TUPLE': 74, - 'CALL': 75, - 'CALL_FUNCTION_EX': 79, - 'CALL_INTRINSIC_1': 80, - 'CALL_INTRINSIC_2': 81, - 'COMPARE_OP': 97, - 'CONTAINS_OP': 101, - 'CONVERT_VALUE': 102, - 'COPY': 103, - 'COPY_FREE_VARS': 104, - 'DELETE_ATTR': 105, - 'DELETE_DEREF': 106, - 'DELETE_FAST': 107, - 'DELETE_GLOBAL': 108, - 'DELETE_NAME': 109, - 'DICT_MERGE': 110, - 'DICT_UPDATE': 111, - 'ENTER_EXECUTOR': 112, - 'EXTENDED_ARG': 113, - 'FOR_ITER': 114, - 'GET_AWAITABLE': 119, - 'IMPORT_FROM': 120, - 'IMPORT_NAME': 121, - 'IS_OP': 122, - 'JUMP_BACKWARD': 123, - 'JUMP_BACKWARD_NO_INTERRUPT': 124, - 'JUMP_FORWARD': 125, - 'KW_NAMES': 126, - 'LIST_APPEND': 127, - 'LIST_EXTEND': 128, - 'LOAD_ATTR': 129, - 'LOAD_CONST': 142, - 'LOAD_DEREF': 143, - 'LOAD_FAST': 144, - 'LOAD_FAST_AND_CLEAR': 145, - 'LOAD_FAST_CHECK': 146, - 'LOAD_FAST_LOAD_FAST': 147, - 'LOAD_FROM_DICT_OR_DEREF': 148, - 'LOAD_FROM_DICT_OR_GLOBALS': 149, - 'LOAD_GLOBAL': 150, - 'LOAD_NAME': 153, - 'LOAD_SUPER_ATTR': 154, - 'MAKE_CELL': 157, - 'MAP_ADD': 158, - 'MATCH_CLASS': 159, - 'POP_JUMP_IF_FALSE': 160, - 'POP_JUMP_IF_NONE': 161, - 'POP_JUMP_IF_NOT_NONE': 162, - 'POP_JUMP_IF_TRUE': 163, - 'RAISE_VARARGS': 164, - 'RERAISE': 165, + 'RETURN_GENERATOR': 48, + 'RETURN_VALUE': 49, + 'SETUP_ANNOTATIONS': 50, + 'STORE_SLICE': 53, + 'STORE_SUBSCR': 54, + 'TO_BOOL': 57, + 'UNARY_INVERT': 64, + 'UNARY_NEGATIVE': 65, + 'UNARY_NOT': 66, + 'WITH_EXCEPT_START': 67, + 'BINARY_OP': 68, + 'BUILD_CONST_KEY_MAP': 69, + 'BUILD_LIST': 70, + 'BUILD_MAP': 71, + 'BUILD_SET': 72, + 'BUILD_SLICE': 73, + 'BUILD_STRING': 74, + 'BUILD_TUPLE': 75, + 'CALL': 76, + 'CALL_FUNCTION_EX': 80, + 'CALL_INTRINSIC_1': 81, + 'CALL_INTRINSIC_2': 82, + 'COMPARE_OP': 98, + 'CONTAINS_OP': 102, + 'CONVERT_VALUE': 103, + 'COPY': 104, + 'COPY_FREE_VARS': 105, + 'DELETE_ATTR': 106, + 'DELETE_DEREF': 107, + 'DELETE_FAST': 108, + 'DELETE_GLOBAL': 109, + 'DELETE_NAME': 110, + 'DICT_MERGE': 111, + 'DICT_UPDATE': 112, + 'ENTER_EXECUTOR': 113, + 'EXTENDED_ARG': 114, + 'FOR_ITER': 115, + 'GET_AWAITABLE': 120, + 'IMPORT_FROM': 121, + 'IMPORT_NAME': 122, + 'IS_OP': 123, + 'JUMP_BACKWARD': 124, + 'JUMP_BACKWARD_NO_INTERRUPT': 125, + 'JUMP_FORWARD': 126, + 'KW_NAMES': 127, + 'LIST_APPEND': 128, + 'LIST_EXTEND': 129, + 'LOAD_ATTR': 130, + 'LOAD_CONST': 143, + 'LOAD_DEREF': 144, + 'LOAD_FAST': 145, + 'LOAD_FAST_AND_CLEAR': 146, + 'LOAD_FAST_CHECK': 147, + 'LOAD_FAST_LOAD_FAST': 148, + 'LOAD_FROM_DICT_OR_DEREF': 149, + 'LOAD_FROM_DICT_OR_GLOBALS': 150, + 'LOAD_GLOBAL': 151, + 'LOAD_NAME': 154, + 'LOAD_SUPER_ATTR': 155, + 'MAKE_CELL': 158, + 'MAP_ADD': 159, + 'MATCH_CLASS': 160, + 'POP_JUMP_IF_FALSE': 161, + 'POP_JUMP_IF_NONE': 162, + 'POP_JUMP_IF_NOT_NONE': 163, + 'POP_JUMP_IF_TRUE': 164, + 'RAISE_VARARGS': 165, 'RESUME': 166, - 'RETURN_CONST': 167, - 'SEND': 168, - 'SET_ADD': 170, - 'SET_FUNCTION_ATTRIBUTE': 171, - 'SET_UPDATE': 172, - 'STORE_ATTR': 173, - 'STORE_DEREF': 175, - 'STORE_FAST': 176, - 'STORE_FAST_LOAD_FAST': 177, - 'STORE_FAST_STORE_FAST': 178, - 'STORE_GLOBAL': 179, - 'STORE_NAME': 180, - 'SWAP': 181, - 'UNPACK_EX': 182, - 'UNPACK_SEQUENCE': 183, - 'YIELD_VALUE': 187, + 'RERAISE': 167, + 'RETURN_CONST': 168, + 'SEND': 169, + 'SET_ADD': 171, + 'SET_FUNCTION_ATTRIBUTE': 172, + 'SET_UPDATE': 173, + 'STORE_ATTR': 174, + 'STORE_DEREF': 176, + 'STORE_FAST': 177, + 'STORE_FAST_LOAD_FAST': 178, + 'STORE_FAST_STORE_FAST': 179, + 'STORE_GLOBAL': 180, + 'STORE_NAME': 181, + 'SWAP': 182, + 'UNPACK_EX': 183, + 'UNPACK_SEQUENCE': 184, + 'YIELD_VALUE': 188, 'INSTRUMENTED_RESUME': 237, 'INSTRUMENTED_END_FOR': 238, 'INSTRUMENTED_END_SEND': 239, @@ -327,4 +331,4 @@ 'STORE_FAST_MAYBE_NULL': 267, } MIN_INSTRUMENTED_OPCODE = 237 -HAVE_ARGUMENT = 67 +HAVE_ARGUMENT = 68 diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index dacd6f6da2c5a9..eae1918efc3506 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -788,7 +788,7 @@ def load_test(x, y=0): return a, b dis_load_test_quickened_code = """\ -%3d 0 RESUME 0 +%3d 0 RESUME_CHECK 0 %3d 2 LOAD_FAST_LOAD_FAST 1 (x, y) 4 STORE_FAST_STORE_FAST 50 (b, a) @@ -805,7 +805,7 @@ def loop_test(): load_test(i) dis_loop_test_quickened_code = """\ -%3d RESUME 0 +%3d RESUME_CHECK 0 %3d BUILD_LIST 0 LOAD_CONST 1 ((1, 2, 3)) @@ -1197,7 +1197,7 @@ def test_super_instructions(self): @requires_specialization def test_binary_specialize(self): binary_op_quicken = """\ - 0 0 RESUME 0 + 0 0 RESUME_CHECK 0 1 2 LOAD_NAME 0 (a) 4 LOAD_NAME 1 (b) @@ -1215,7 +1215,7 @@ def test_binary_specialize(self): self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)", True) binary_subscr_quicken = """\ - 0 0 RESUME 0 + 0 0 RESUME_CHECK 0 1 2 LOAD_NAME 0 (a) 4 LOAD_CONST 0 (0) @@ -1236,7 +1236,7 @@ def test_binary_specialize(self): @requires_specialization def test_load_attr_specialize(self): load_attr_quicken = """\ - 0 0 RESUME 0 + 0 0 RESUME_CHECK 0 1 2 LOAD_CONST 0 ('a') 4 LOAD_ATTR_SLOT 0 (__class__) @@ -1251,7 +1251,7 @@ def test_load_attr_specialize(self): @requires_specialization def test_call_specialize(self): call_quicken = """\ - 0 RESUME 0 + 0 RESUME_CHECK 0 1 LOAD_NAME 0 (str) PUSH_NULL @@ -1640,197 +1640,197 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=158, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=158, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=74, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=75, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, is_jump_target=False, positions=None), Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=69, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=70, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=172, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=172, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=177, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=70, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=71, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=49, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, is_jump_target=False, positions=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=104, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=105, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=158, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=158, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=74, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=75, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, is_jump_target=False, positions=None), Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=172, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=172, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=177, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=49, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=104, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=105, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=143, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=147, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=144, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=148, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=168, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), Instruction(opname='GET_ITER', opcode=31, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=114, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=115, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=177, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=True, line_number=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=True, line_number=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=98, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=161, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=False, line_number=5, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=124, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=True, line_number=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=True, line_number=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=98, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=164, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=False, line_number=7, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=124, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=False, line_number=7, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=True, line_number=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=125, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=False, line_number=8, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=126, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=False, line_number=8, is_jump_target=False, positions=None), Instruction(opname='END_FOR', opcode=24, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=True, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=False, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=False, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=True, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=False, line_number=10, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=False, line_number=10, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=False, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=146, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=True, line_number=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=True, line_number=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=False, line_number=12, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=12, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=147, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=True, line_number=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=57, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=161, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=True, line_number=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=False, line_number=12, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=12, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=True, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=False, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=67, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=False, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=False, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=True, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=False, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=False, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=False, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=True, line_number=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=True, line_number=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=False, line_number=16, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=False, line_number=16, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=False, line_number=16, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=125, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=True, line_number=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=True, line_number=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=True, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=False, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=68, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=False, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=177, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=False, line_number=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=True, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=False, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=98, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=False, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=161, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=False, line_number=14, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=124, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=True, line_number=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=True, line_number=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=False, line_number=16, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=98, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=False, line_number=16, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=161, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=False, line_number=16, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=126, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=True, line_number=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=True, line_number=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=57, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=161, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=124, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=False, line_number=11, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=False, line_number=19, is_jump_target=False, positions=None), Instruction(opname='NOP', opcode=42, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=True, line_number=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=True, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=False, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=67, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=False, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=True, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=False, line_number=21, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=68, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=False, line_number=21, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=True, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=145, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=True, line_number=25, is_jump_target=False, positions=None), Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=176, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=True, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=False, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=177, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=True, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=False, line_number=26, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=26, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=True, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=True, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=True, line_number=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=True, line_number=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=28, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=168, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=False, line_number=28, is_jump_target=False, positions=None), Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=True, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=66, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=67, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='TO_BOOL', opcode=57, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=164, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=167, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, is_jump_target=True, positions=None), Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=25, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=124, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=False, line_number=25, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=104, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=True, line_number=None, is_jump_target=False, positions=None), Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=167, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=None, is_jump_target=False, positions=None), Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=True, line_number=22, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=True, line_number=22, is_jump_target=False, positions=None), Instruction(opname='CHECK_EXC_MATCH', opcode=20, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=22, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=False, line_number=22, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=161, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=False, line_number=22, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=22, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=True, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=True, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=23, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=23, is_jump_target=False, positions=None), Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=123, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=True, line_number=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=True, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=124, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=False, line_number=23, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=167, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=True, line_number=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=104, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=True, line_number=None, is_jump_target=False, positions=None), Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=167, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=None, is_jump_target=False, positions=None), Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=True, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=151, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=True, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=143, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=76, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=28, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=167, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=104, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=28, is_jump_target=False, positions=None), Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=28, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=167, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=28, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, is_jump_target=False), + Instruction(opname='RETURN_CONST', opcode=168, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, is_jump_target=False), ] diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-26-10-36-45.gh-issue-108614.wl5l-W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-26-10-36-45.gh-issue-108614.wl5l-W.rst new file mode 100644 index 00000000000000..ace670c9ba7fdf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-26-10-36-45.gh-issue-108614.wl5l-W.rst @@ -0,0 +1,2 @@ +Add RESUME_CHECK instruction, to avoid having to handle instrumentation, +signals, and contexts switches in the tier 2 execution engine. diff --git a/Objects/genobject.c b/Objects/genobject.c index 10c55efb1b1e9b..40033d10bea9ee 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -333,7 +333,11 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { - return instr->op.code == RESUME || instr->op.code == INSTRUMENTED_RESUME; + return ( + instr->op.code == RESUME || + instr->op.code == RESUME_CHECK || + instr->op.code == INSTRUMENTED_RESUME + ); } static inline bool diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 3fd6cdade69f9e..7590e87ab1eadd 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,17 +1,17 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,164,0,0,0,166,0,142,0,142,1, - 121,0,180,0,142,0,142,1,121,1,180,1,153,2,46,0, - 142,2,75,1,0,0,0,0,0,0,44,0,153,2,46,0, - 142,3,153,0,129,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,75,2,0,0,0,0,0,0, - 44,0,153,1,129,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,46,0,75,0,0,0,0,0, - 0,0,142,4,12,0,0,0,180,5,142,5,31,0,114,20, - 0,0,180,6,153,2,46,0,142,6,153,6,27,0,142,7, - 153,5,153,6,12,0,0,0,27,0,73,4,75,1,0,0, - 0,0,0,0,44,0,123,22,0,0,24,0,167,1,41,8, + 0,0,0,0,0,243,164,0,0,0,166,0,143,0,143,1, + 122,0,181,0,143,0,143,1,122,1,181,1,154,2,46,0, + 143,2,76,1,0,0,0,0,0,0,44,0,154,2,46,0, + 143,3,154,0,130,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,76,2,0,0,0,0,0,0, + 44,0,154,1,130,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,46,0,76,0,0,0,0,0, + 0,0,143,4,12,0,0,0,181,5,143,5,31,0,115,20, + 0,0,181,6,154,2,46,0,143,6,154,6,27,0,143,7, + 154,5,154,6,12,0,0,0,27,0,74,4,76,1,0,0, + 0,0,0,0,44,0,124,22,0,0,24,0,168,1,41,8, 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, 101,108,108,111,32,87,111,114,108,100,122,8,115,121,115,46, 97,114,103,118,218,6,99,111,110,102,105,103,41,5,218,12, diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 07e8a711575c5b..11fbf4443dda16 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -7,7 +7,7 @@ break; } - case RESUME: { + case RESUME_CHECK: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fae1da31875d66..8820b52774671b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -132,24 +132,37 @@ dummy_func( inst(NOP, (--)) { } + family(RESUME, 0) = { + RESUME_CHECK, + }; + inst(RESUME, (--)) { + TIER_ONE_ONLY assert(frame == tstate->current_frame); - /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); - #if TIER_ONE next_instr--; - #endif - #if TIER_TWO - goto deoptimize; - #endif } - else if (oparg < 2) { - CHECK_EVAL_BREAKER(); + else { + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } + next_instr[-1].op.code = RESUME_CHECK; } } + inst(RESUME_CHECK, (--)) { +#if defined(__EMSCRIPTEN__) + DEOPT_IF(emscripten_signal_clock == 0, RESUME); + emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + /* Possibly combine these two checks */ + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version + != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + } + inst(INSTRUMENTED_RESUME, (--)) { /* Possible performance enhancement: * We need to check the eval breaker anyway, can we diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 81fbb7982ad11c..f5d915554d56e7 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -374,6 +374,8 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } +/* Marker to specify tier 1 only instructions */ +#define TIER_ONE_ONLY /* Implementation of "macros" that modify the instruction pointer, * stack pointer, or frame pointer. diff --git a/Python/emscripten_signal.c b/Python/emscripten_signal.c index d617ddfeb37c5a..1a196385b8ce34 100644 --- a/Python/emscripten_signal.c +++ b/Python/emscripten_signal.c @@ -38,19 +38,17 @@ _Py_CheckEmscriptenSignals(void) } } - #define PY_EMSCRIPTEN_SIGNAL_INTERVAL 50 static int emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; void _Py_CheckEmscriptenSignalsPeriodically(void) { - if (!Py_EMSCRIPTEN_SIGNAL_HANDLING) { - return; - } - emscripten_signal_clock--; if (emscripten_signal_clock == 0) { emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; _Py_CheckEmscriptenSignals(); } + else if (Py_EMSCRIPTEN_SIGNAL_HANDLING) { + emscripten_signal_clock--; + } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0d5606f5a44efe..f4c526a5a8c0a2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7,22 +7,15 @@ break; } - case RESUME: { - assert(frame == tstate->current_frame); - /* Possibly combine this with eval breaker */ - if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - #if TIER_ONE - next_instr--; - #endif - #if TIER_TWO - goto deoptimize; - #endif - } - else if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } + case RESUME_CHECK: { +#if defined(__EMSCRIPTEN__) + DEOPT_IF(emscripten_signal_clock == 0, RESUME); + emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + /* Possibly combine these two checks */ + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version + != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 34cea84245f591..84f83db128ea50 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,24 +8,36 @@ } TARGET(RESUME) { + PREDICTED(RESUME); + static_assert(0 == 0, "incorrect cache size"); + TIER_ONE_ONLY assert(frame == tstate->current_frame); - /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; - #if TIER_ONE next_instr--; - #endif - #if TIER_TWO - goto deoptimize; - #endif } - else if (oparg < 2) { - CHECK_EVAL_BREAKER(); + else { + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } + next_instr[-1].op.code = RESUME_CHECK; } DISPATCH(); } + TARGET(RESUME_CHECK) { +#if defined(__EMSCRIPTEN__) + DEOPT_IF(emscripten_signal_clock == 0, RESUME); + emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + /* Possibly combine these two checks */ + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version + != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + DISPATCH(); + } + TARGET(INSTRUMENTED_RESUME) { /* Possible performance enhancement: * We need to check the eval breaker anyway, can we diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 305eb0bfe2a7c4..413bb884dcf752 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -46,6 +46,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_TOP, &&TARGET_PUSH_EXC_INFO, &&TARGET_PUSH_NULL, + &&TARGET_RESUME_CHECK, &&TARGET_RETURN_GENERATOR, &&TARGET_RETURN_VALUE, &&TARGET_SETUP_ANNOTATIONS, @@ -164,8 +165,8 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_RAISE_VARARGS, - &&TARGET_RERAISE, &&TARGET_RESUME, + &&TARGET_RERAISE, &&TARGET_RETURN_CONST, &&TARGET_SEND, &&TARGET_SEND_GEN, @@ -235,7 +236,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 9143ae0db7be81..145c1ade839b82 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -37,6 +37,7 @@ class ActiveCacheEffect: "import_from", "import_name", "_PyObject_CallNoArgs", # Proxy for BEFORE_WITH + "TIER_ONE_ONLY", ) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 25de3a5adc2cf9..25be5ca3e0da5d 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -362,7 +362,8 @@ def family_def(self) -> Family | None: if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.COMMA): if not (size := self.expect(lx.IDENTIFIER)): - raise self.make_syntax_error("Expected identifier") + if not (size := self.expect(lx.NUMBER)): + raise self.make_syntax_error("Expected identifier or number") if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if not self.expect(lx.LBRACE): From 891236f48263e2d4c650b7a127fc9bffb8327807 Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Thu, 7 Sep 2023 06:52:58 -0700 Subject: [PATCH 547/598] gh-71770: Add more details on behavior of configparser's default_section (#31562) Co-authored-by: Hugo van Kemenade --- Doc/library/configparser.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index a7f75fd6e84f4c..bb282428c5fffc 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -935,8 +935,10 @@ ConfigParser Objects When *default_section* is given, it specifies the name for the special section holding default values for other sections and interpolation purposes - (normally named ``"DEFAULT"``). This value can be retrieved and changed on - runtime using the ``default_section`` instance attribute. + (normally named ``"DEFAULT"``). This value can be retrieved and changed at + runtime using the ``default_section`` instance attribute. This won't + re-evaluate an already parsed config file, but will be used when writing + parsed settings to a new config file. Interpolation behaviour may be customized by providing a custom handler through the *interpolation* argument. ``None`` can be used to turn off From ac31f714c3e55a7951a9f3f9c823740c20c5d595 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 7 Sep 2023 16:53:33 +0300 Subject: [PATCH 548/598] gh-107544: Add docs about `json.dumps(..., default=)` (#108259) --- Doc/library/json.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 6c3059381776c9..b337b5f9960e8e 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -54,12 +54,23 @@ Compact encoding:: Pretty printing:: >>> import json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) + >>> print(json.dumps({'6': 7, '4': 5}, sort_keys=True, indent=4)) { "4": 5, "6": 7 } +Specializing JSON object encoding:: + + >>> import json + >>> def custom_json(obj): + ... if isinstance(obj, complex): + ... return {'__complex__': True, 'real': obj.real, 'imag': obj.imag} + ... raise TypeError(f'Cannot serialize object of {type(obj)}') + ... + >>> json.dumps(1 + 2j, default=custom_json) + '{"__complex__": true, "real": 1.0, "imag": 2.0}' + Decoding JSON:: >>> import json From b2729e93e9d73503b1fda4ea4fecd77c58909091 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 17:00:13 +0300 Subject: [PATCH 549/598] gh-88943: Improve syntax error for non-ASCII character that follows a numerical literal (GH-109081) It now points on the invalid non-ASCII character, not on the valid numerical literal. --- Lib/test/test_grammar.py | 4 ++++ .../2023-09-07-16-05-36.gh-issue-88943.rH_X3W.rst | 3 +++ Parser/tokenizer.c | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-07-16-05-36.gh-issue-88943.rH_X3W.rst diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 7c15a23a69163b..8501006b799262 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -236,6 +236,10 @@ def check(test, error=False): check(f"[{num}for x in ()]") check(f"{num}spam", error=True) + # gh-88943: Invalid non-ASCII character following a numerical literal. + with self.assertRaisesRegex(SyntaxError, r"invalid character '⁄' \(U\+2044\)"): + compile(f"{num}⁄7", "", "eval") + with self.assertWarnsRegex(SyntaxWarning, r'invalid \w+ literal'): compile(f"{num}is x", "", "eval") with warnings.catch_warnings(): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-07-16-05-36.gh-issue-88943.rH_X3W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-16-05-36.gh-issue-88943.rH_X3W.rst new file mode 100644 index 00000000000000..a99830fe4227c9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-16-05-36.gh-issue-88943.rH_X3W.rst @@ -0,0 +1,3 @@ +Improve syntax error for non-ASCII character that follows a numerical +literal. It now points on the invalid non-ASCII character, not on the valid +numerical literal. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 6ec24895785568..46b7159ff0516b 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1642,7 +1642,7 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { tok_nextc(tok); } else /* In future releases, only error will remain. */ - if (is_potential_identifier_char(c)) { + if (c < 128 && is_potential_identifier_char(c)) { tok_backup(tok, c); syntaxerror(tok, "invalid %s literal", kind); return 0; From b72251de930c8ec6893f1b3f6fdf1640cc17dfed Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Thu, 7 Sep 2023 15:20:33 +0100 Subject: [PATCH 550/598] gh-102823: Document return type of floor division on floats (#102824) Co-authored-by: Hugo van Kemenade --- Doc/library/stdtypes.rst | 8 +++++--- .../2023-03-19-09-39-31.gh-issue-102823.OzsOz0.rst | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-03-19-09-39-31.gh-issue-102823.OzsOz0.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 6c07ee585480e9..652ed147af8422 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -282,7 +282,7 @@ the operations, see :ref:`operator-summary`): +---------------------+---------------------------------+---------+--------------------+ | ``x / y`` | quotient of *x* and *y* | | | +---------------------+---------------------------------+---------+--------------------+ -| ``x // y`` | floored quotient of *x* and | \(1) | | +| ``x // y`` | floored quotient of *x* and | \(1)\(2)| | | | *y* | | | +---------------------+---------------------------------+---------+--------------------+ | ``x % y`` | remainder of ``x / y`` | \(2) | | @@ -319,8 +319,10 @@ the operations, see :ref:`operator-summary`): Notes: (1) - Also referred to as integer division. The resultant value is a whole - integer, though the result's type is not necessarily int. The result is + Also referred to as integer division. For operands of type :class:`int`, + the result has type :class:`int`. For operands of type :class:`float`, + the result has type :class:`float`. In general, the result is a whole + integer, though the result's type is not necessarily :class:`int`. The result is always rounded towards minus infinity: ``1//2`` is ``0``, ``(-1)//2`` is ``-1``, ``1//(-2)`` is ``-1``, and ``(-1)//(-2)`` is ``0``. diff --git a/Misc/NEWS.d/next/Documentation/2023-03-19-09-39-31.gh-issue-102823.OzsOz0.rst b/Misc/NEWS.d/next/Documentation/2023-03-19-09-39-31.gh-issue-102823.OzsOz0.rst new file mode 100644 index 00000000000000..1e32f3c89231c8 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-03-19-09-39-31.gh-issue-102823.OzsOz0.rst @@ -0,0 +1,2 @@ +Document the return type of ``x // y`` when ``x`` and ``y`` have type +:class:`float`. From f2584eade378910b9ea18072bb1dab3dd58e23bb Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 7 Sep 2023 08:56:43 -0600 Subject: [PATCH 551/598] gh-108732: include comprehension locals in frame.f_locals (#109026) Co-authored-by: Radislav Chugunov <52372310+chgnrdv@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- Lib/test/test_listcomps.py | 7 +++++++ .../2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst | 2 ++ Objects/frameobject.c | 14 ++++++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index bedd99b4a44fcb..c1089574d71b02 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -596,6 +596,13 @@ def test_exception_in_post_comp_call(self): """ self._check_in_scopes(code, {"value": [1, None]}) + def test_frame_locals(self): + code = """ + val = [sys._getframe().f_locals for a in [0]][0]["a"] + """ + import sys + self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + __test__ = {'doctests' : doctests} diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst new file mode 100644 index 00000000000000..94a143b86b6708 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-06-13-28-42.gh-issue-108732.I6DkEQ.rst @@ -0,0 +1,2 @@ +Make iteration variables of module- and class-scoped comprehensions visible +to pdb and other tools that use ``frame.f_locals`` again. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7017cc6c0e295f..53764a41ca010a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -25,10 +25,16 @@ static PyMemberDef frame_memberlist[] = { static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { - if (PyFrame_FastToLocalsWithError(f) < 0) + if (f == NULL) { + PyErr_BadInternalCall(); return NULL; - PyObject *locals = f->f_frame->f_locals; - return Py_NewRef(locals); + } + assert(!_PyFrame_IsIncomplete(f->f_frame)); + PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); + if (locals) { + f->f_fast_as_locals = 1; + } + return locals; } int @@ -1342,11 +1348,11 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); if (f == NULL) { PyErr_BadInternalCall(); return -1; } + assert(!_PyFrame_IsIncomplete(f->f_frame)); int err = _PyFrame_FastToLocalsWithError(f->f_frame); if (err == 0) { f->f_fast_as_locals = 1; From 403ab1306a6e9860197bce57eadcb83418966f21 Mon Sep 17 00:00:00 2001 From: Christoph Anton Mitterer Date: Thu, 7 Sep 2023 17:50:10 +0200 Subject: [PATCH 552/598] gh-107924: re-order os.sendfile() flag documentation (#107926) Co-authored-by: Hugo van Kemenade --- Doc/library/os.rst | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4f4d8c99023529..c67b966f777db8 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1618,25 +1618,6 @@ or `the MSDN `_ on Windo Parameters *out* and *in* was renamed to *out_fd* and *in_fd*. -.. function:: set_blocking(fd, blocking, /) - - Set the blocking mode of the specified file descriptor. Set the - :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. - - See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - - .. availability:: Unix, Windows. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - On Windows, this function is limited to pipes. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.12 - Added support for pipes on Windows. - .. data:: SF_NODISKIO SF_MNOWAIT SF_SYNC @@ -1658,6 +1639,26 @@ or `the MSDN `_ on Windo .. versionadded:: 3.11 +.. function:: set_blocking(fd, blocking, /) + + Set the blocking mode of the specified file descriptor. Set the + :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. + + See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. + + .. availability:: Unix, Windows. + + The function is limited on Emscripten and WASI, see + :ref:`wasm-availability` for more information. + + On Windows, this function is limited to pipes. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.12 + Added support for pipes on Windows. + + .. function:: splice(src, dst, count, offset_src=None, offset_dst=None) Transfer *count* bytes from file descriptor *src*, starting from offset From 96396962ce7a83c09bac5061121c779ca6b25ef5 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 7 Sep 2023 18:23:11 +0100 Subject: [PATCH 553/598] gh-109094: remove unnecessary updates of frame->prev_instr in instrumentation functions (#109076) --- Lib/test/test_sys_settrace.py | 9 +++++++++ Python/instrumentation.c | 8 ++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 232900876392ff..369a276ac33a12 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -317,6 +317,13 @@ def generator_example(): [(5, 'line'), (5, 'return')]) +def lineno_matches_lasti(frame): + last_line = None + for start, end, line in frame.f_code.co_lines(): + if start <= frame.f_lasti < end: + last_line = line + return last_line == frame.f_lineno + class Tracer: def __init__(self, trace_line_events=None, trace_opcode_events=None): self.trace_line_events = trace_line_events @@ -330,6 +337,7 @@ def _reconfigure_frame(self, frame): frame.f_trace_opcodes = self.trace_opcode_events def trace(self, frame, event, arg): + assert lineno_matches_lasti(frame) self._reconfigure_frame(frame) self.events.append((frame.f_lineno, event)) return self.trace @@ -1890,6 +1898,7 @@ def __init__(self, function, jumpFrom, jumpTo, event='line', def trace(self, frame, event, arg): if self.done: return + assert lineno_matches_lasti(frame) # frame.f_code.co_firstlineno is the first line of the decorator when # 'function' is decorated and the decorator may be written using # multiple physical lines when it is too long. Use the first line diff --git a/Python/instrumentation.c b/Python/instrumentation.c index acc3278d50a60a..74c3adf0c1f475 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1057,8 +1057,6 @@ _Py_call_instrumentation_jump( assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); assert(frame->prev_instr == instr); - /* Event should occur after the jump */ - frame->prev_instr = target; PyCodeObject *code = _PyFrame_GetCode(frame); int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1071,12 +1069,10 @@ _Py_call_instrumentation_jump( if (err) { return NULL; } - if (frame->prev_instr != target) { + if (frame->prev_instr != instr) { /* The callback has caused a jump (by setting the line number) */ return frame->prev_instr; } - /* Reset prev_instr for INSTRUMENTED_LINE */ - frame->prev_instr = instr; return target; } @@ -1125,7 +1121,7 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { - frame->prev_instr = instr; + assert(frame->prev_instr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); From f9f085c326cdaa34ebb3ca018228a63825b12122 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 20:53:38 +0300 Subject: [PATCH 554/598] gh-103186: Make test_generated_cases less noisy by default (GH-109100) Print additional details only when tests are run with -vv. --- Lib/test/test_generated_cases.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ed56f95af99417..be3dfd204ee143 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1,7 +1,9 @@ +import contextlib import tempfile import unittest import os +from test import support from test import test_tools test_tools.skip_if_missing('cases_generator') @@ -12,6 +14,12 @@ from parsing import StackEffect +def handle_stderr(): + if support.verbose > 1: + return contextlib.nullcontext() + else: + return support.captured_stderr() + class TestEffects(unittest.TestCase): def test_effect_sizes(self): input_effects = [ @@ -81,11 +89,12 @@ def run_cases_test(self, input: str, expected: str): temp_input.flush() a = generate_cases.Generator([self.temp_input_filename]) - a.parse() - a.analyze() - if a.errors: - raise RuntimeError(f"Found {a.errors} errors") - a.write_instructions(self.temp_output_filename, False) + with handle_stderr(): + a.parse() + a.analyze() + if a.errors: + raise RuntimeError(f"Found {a.errors} errors") + a.write_instructions(self.temp_output_filename, False) with open(self.temp_output_filename) as temp_output: lines = temp_output.readlines() From 1829a3c9a3712b6a68a3a449e4a08787c73da51d Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Thu, 7 Sep 2023 14:13:32 -0400 Subject: [PATCH 555/598] gh-75743: Restore test_timeout.testConnectTimeout() (#109087) This un-skips this test now that pythontest.net implements appropriate firewall rules for it. --- Lib/test/test_timeout.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_timeout.py b/Lib/test/test_timeout.py index 30e843a423a777..35ff56f1a5ee09 100644 --- a/Lib/test/test_timeout.py +++ b/Lib/test/test_timeout.py @@ -148,13 +148,12 @@ def setUp(self): def tearDown(self): self.sock.close() - @unittest.skipIf(True, 'need to replace these hosts; see bpo-35518') def testConnectTimeout(self): # Testing connect timeout is tricky: we need to have IP connectivity # to a host that silently drops our packets. We can't simulate this # from Python because it's a function of the underlying TCP/IP stack. - # So, the following Snakebite host has been defined: - blackhole = resolve_address('blackhole.snakebite.net', 56666) + # So, the following port on the pythontest.net host has been defined: + blackhole = resolve_address('pythontest.net', 56666) # Blackhole has been configured to silently drop any incoming packets. # No RSTs (for TCP) or ICMP UNREACH (for UDP/ICMP) will be sent back @@ -166,7 +165,7 @@ def testConnectTimeout(self): # to firewalling or general network configuration. In order to improve # our confidence in testing the blackhole, a corresponding 'whitehole' # has also been set up using one port higher: - whitehole = resolve_address('whitehole.snakebite.net', 56667) + whitehole = resolve_address('pythontest.net', 56667) # This address has been configured to immediately drop any incoming # packets as well, but it does it respectfully with regards to the @@ -180,20 +179,15 @@ def testConnectTimeout(self): # timeframe). # For the records, the whitehole/blackhole configuration has been set - # up using the 'pf' firewall (available on BSDs), using the following: + # up using the 'iptables' firewall, using the following rules: # - # ext_if="bge0" - # - # blackhole_ip="35.8.247.6" - # whitehole_ip="35.8.247.6" - # blackhole_port="56666" - # whitehole_port="56667" - # - # block return in log quick on $ext_if proto { tcp udp } \ - # from any to $whitehole_ip port $whitehole_port - # block drop in log quick on $ext_if proto { tcp udp } \ - # from any to $blackhole_ip port $blackhole_port + # -A INPUT -p tcp --destination-port 56666 -j DROP + # -A INPUT -p udp --destination-port 56666 -j DROP + # -A INPUT -p tcp --destination-port 56667 -j REJECT + # -A INPUT -p udp --destination-port 56667 -j REJECT # + # See https://github.com/python/psf-salt/blob/main/pillar/base/firewall/snakebite.sls + # for the current configuration. skip = True sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) From 7e1a7abb9831965cdec477e62dbe4f8415b8a582 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 21:28:18 +0300 Subject: [PATCH 556/598] gh-68403: Fix test_coverage in test_trace (GH-108910) Its behavior no longer affected by test running options such as -m. --- Lib/test/test_trace.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index d1ef005a4314ed..c1e289bcaff9e5 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -360,9 +360,14 @@ def tearDown(self): rmtree(TESTFN) unlink(TESTFN) - def _coverage(self, tracer, - cmd='import test.support, test.test_pprint;' - 'test.support.run_unittest(test.test_pprint.QueryTestCase)'): + DEFAULT_SCRIPT = '''if True: + import unittest + from test.test_pprint import QueryTestCase + loader = unittest.TestLoader() + tests = loader.loadTestsFromTestCase(QueryTestCase) + tests(unittest.TestResult()) + ''' + def _coverage(self, tracer, cmd=DEFAULT_SCRIPT): tracer.run(cmd) r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=TESTFN) From 74fc96bc60f5c02bde50ff2f3516add99483e402 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 7 Sep 2023 11:34:18 -0700 Subject: [PATCH 557/598] Add version directives to ast docs (#108788) --- Doc/library/ast.rst | 46 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 8f5148feb809a3..10b3e2dae472e1 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -650,10 +650,10 @@ Expressions .. class:: NamedExpr(target, value) - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. .. doctest:: @@ -663,6 +663,7 @@ Expressions target=Name(id='x', ctx=Store()), value=Constant(value=4))) + .. versionadded:: 3.8 Subscripting ~~~~~~~~~~~~ @@ -1036,6 +1037,7 @@ Statements value=Name(id='int', ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 Other statements which are only applicable inside functions or loops are described in other sections. @@ -1318,6 +1320,7 @@ Control flow finalbody=[])], type_ignores=[]) + .. versionadded:: 3.11 .. class:: ExceptHandler(type, name, body) @@ -1407,6 +1410,8 @@ Pattern matching that is being matched against the cases) and ``cases`` contains an iterable of :class:`match_case` nodes with the different cases. + .. versionadded:: 3.10 + .. class:: match_case(pattern, guard, body) A single case pattern in a ``match`` statement. ``pattern`` contains the @@ -1458,6 +1463,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchValue(value) A match literal or value pattern that compares by equality. ``value`` is @@ -1485,6 +1492,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchSingleton(value) A match literal pattern that compares by identity. ``value`` is the @@ -1510,6 +1519,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchSequence(patterns) A match sequence pattern. ``patterns`` contains the patterns to be matched @@ -1541,6 +1552,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchStar(name) Matches the rest of the sequence in a variable length match sequence pattern. @@ -1581,6 +1594,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchMapping(keys, patterns, rest) A match mapping pattern. ``keys`` is a sequence of expression nodes. @@ -1627,6 +1642,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) A match class pattern. ``cls`` is an expression giving the nominal class to @@ -1691,6 +1708,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchAs(pattern, name) A match "as-pattern", capture pattern or wildcard pattern. ``pattern`` @@ -1732,6 +1751,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchOr(patterns) A match "or-pattern". An or-pattern matches each of its subpatterns in turn @@ -1764,6 +1785,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. _ast-type-params: Type parameters @@ -1795,6 +1818,8 @@ aliases. ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 + .. class:: ParamSpec(name) A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. @@ -1818,6 +1843,8 @@ aliases. ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 + .. class:: TypeVarTuple(name) A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. @@ -1842,6 +1869,8 @@ aliases. ctx=Load()))], type_ignores=[]) + .. versionadded:: 3.12 + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1861,6 +1890,9 @@ Function and class definitions ``type_comment`` is an optional string with the type annotation as a comment. + .. versionchanged:: 3.12 + Added ``type_params``. + .. class:: Lambda(args, body) @@ -2059,6 +2091,9 @@ Function and class definitions type_params=[])], type_ignores=[]) + .. versionchanged:: 3.12 + Added ``type_params``. + Async and await ^^^^^^^^^^^^^^^ @@ -2067,6 +2102,9 @@ Async and await An ``async def`` function definition. Has the same fields as :class:`FunctionDef`. + .. versionchanged:: 3.12 + Added ``type_params``. + .. class:: Await(value) From b9831e5c98de280870b6d932033b868ef56fa2fa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Sep 2023 23:08:55 +0300 Subject: [PATCH 558/598] Use unittest test runner for doctests in test_statistics (GH-108921) --- Lib/test/test_statistics.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 23a3973305303d..f9b0ac2ad7b116 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -698,14 +698,6 @@ def test_check_all(self): 'missing name "%s" in __all__' % name) -class DocTests(unittest.TestCase): - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -OO and above") - def test_doc_tests(self): - failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) - self.assertGreater(tried, 0) - self.assertEqual(failed, 0) - class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): errmsg = ( @@ -3145,6 +3137,7 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) + tests.addTests(doctest.DocTestSuite(statistics)) return tests From c74e440168fab9bf91346471087a394af13fa2db Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 7 Sep 2023 18:19:03 -0700 Subject: [PATCH 559/598] gh-109022: [Enum] require `names=()` to create empty enum type (GH-109048) add guard so that ``Enum('bar')`` raises a TypeError instead of creating a new enum class called `bar`. To create the new but empty class, use: huh = Enum('bar', names=()) --- Lib/enum.py | 5 +++++ Lib/test/test_enum.py | 11 +++++++---- .../2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst | 2 ++ 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst diff --git a/Lib/enum.py b/Lib/enum.py index 4b99e7bda2cca5..994a7b9c73f9a7 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -730,6 +730,11 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type + if names is None and type is None: + # no body? no data-type? possibly wrong usage + raise TypeError( + f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" + ) return cls._create_( class_name=value, names=names, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a838b93341a608..8c1f285f7b3bc2 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -316,6 +316,7 @@ def __str__(self): return self.name.title() def __format__(self, spec): return ''.join(reversed(self.name)) + self.NewBaseEnum = NewBaseEnum class NewSubEnum(NewBaseEnum): first = auto() self.NewSubEnum = NewSubEnum @@ -382,10 +383,8 @@ def __str__(self): return self.name.title() def __format__(self, spec): return ''.join(reversed(self.name)) - NewBaseEnum = self.enum_type('NewBaseEnum', dict(__format__=__format__, __str__=__str__)) - class NewSubEnum(NewBaseEnum): - first = auto() - self.NewSubEnum = NewBaseEnum('NewSubEnum', 'first') + self.NewBaseEnum = self.enum_type('NewBaseEnum', dict(__format__=__format__, __str__=__str__)) + self.NewSubEnum = self.NewBaseEnum('NewSubEnum', 'first') # def _generate_next_value_(name, start, last, values): pass @@ -601,6 +600,10 @@ class SubEnum(SuperEnum): self.assertTrue('description' not in dir(SubEnum)) self.assertTrue('description' in dir(SubEnum.sample), dir(SubEnum.sample)) + def test_empty_enum_has_no_values(self): + with self.assertRaisesRegex(TypeError, "<.... 'NewBaseEnum'> has no members"): + self.NewBaseEnum(7) + def test_enum_in_enum_out(self): Main = self.MainEnum self.assertIs(Main(Main.first), Main.first) diff --git a/Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst b/Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst new file mode 100644 index 00000000000000..8c13d43ee9744b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-06-19-33-41.gh-issue-108682.35Xnc5.rst @@ -0,0 +1,2 @@ +Enum: require ``names=()`` or ``type=...`` to create an empty enum using +the functional syntax. From 00cf626cd41f806062c22a913b647b4efa84c476 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 7 Sep 2023 22:37:29 -0700 Subject: [PATCH 560/598] Update `CODEOWNERS` for `Tools/wasm/` (#109119) --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 578cd71a7bd211..81c580eb778625 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -179,3 +179,6 @@ Doc/c-api/stable.rst @encukou /Tools/clinic/** @erlend-aasland @AlexWaygood /Lib/test/test_clinic.py @erlend-aasland @AlexWaygood Doc/howto/clinic.rst @erlend-aasland + +# WebAssembly +/Tools/wasm/ @brettcannon From 15d4c9fabce67b8a1b5bd9dec9612014ec18291a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 8 Sep 2023 10:34:40 +0100 Subject: [PATCH 561/598] GH-108716: Turn off deep-freezing of code objects. (GH-108722) --- Include/cpython/import.h | 1 - Include/internal/pycore_code.h | 2 - Include/internal/pycore_interp.h | 1 + Lib/test/test_ctypes/test_values.py | 1 - Makefile.pre.in | 1 - ...-08-28-03-38-28.gh-issue-108716.HJBPwt.rst | 2 + Objects/codeobject.c | 7 ++- Objects/funcobject.c | 9 +-- Programs/_bootstrap_python.c | 2 - Programs/_freeze_module.c | 2 - Python/frozen.c | 58 +++++++++---------- Python/import.c | 13 +---- Python/pylifecycle.c | 6 -- Python/pystate.c | 1 + Tools/build/freeze_modules.py | 30 +++------- 15 files changed, 50 insertions(+), 86 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst diff --git a/Include/cpython/import.h b/Include/cpython/import.h index cdfdd15bfa48d2..7daf0b84fcf71b 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -17,7 +17,6 @@ struct _frozen { const unsigned char *code; int size; int is_package; - PyObject *(*get_code)(void); }; /* Embedding apps may change this pointer to point to their favorite diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b3f480c7204172..257b0583edae11 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -465,8 +465,6 @@ adaptive_counter_backoff(uint16_t counter) { return adaptive_counter_bits(value, backoff); } -extern uint32_t _Py_next_func_version; - /* Comparison bit masks. */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index e0b7a325929257..ba5764e943e676 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -186,6 +186,7 @@ struct _is { _PyOptimizerObject *optimizer; uint16_t optimizer_resume_threshold; uint16_t optimizer_backedge_threshold; + uint32_t next_func_version; _Py_GlobalMonitors monitors; bool f_opcode_trace_set; diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index 9f8b69409cb880..d0b4803dff8529 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -58,7 +58,6 @@ class struct_frozen(Structure): ("code", POINTER(c_ubyte)), ("size", c_int), ("is_package", c_int), - ("get_code", POINTER(c_ubyte)), # Function ptr ] FrozenTable = POINTER(struct_frozen) diff --git a/Makefile.pre.in b/Makefile.pre.in index aa3eaabc7559f6..19a802997838a4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -505,7 +505,6 @@ LIBRARY_OBJS_OMIT_FROZEN= \ LIBRARY_OBJS= \ $(LIBRARY_OBJS_OMIT_FROZEN) \ - $(DEEPFREEZE_OBJS) \ Modules/getpath.o \ Python/frozen.o diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst new file mode 100644 index 00000000000000..f63eb8689d63a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-28-03-38-28.gh-issue-108716.HJBPwt.rst @@ -0,0 +1,2 @@ +Turn off deep-freezing of code objects. Modules are still frozen, so that a +file system search is not needed for common modules. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1443027ff293ac..d00bd0422f004d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -427,9 +427,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->co_version = _Py_next_func_version; - if (_Py_next_func_version != 0) { - _Py_next_func_version++; + PyInterpreterState *interp = _PyInterpreterState_GET(); + co->co_version = interp->next_func_version; + if (interp->next_func_version != 0) { + interp->next_func_version++; } co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 648b660859c04d..231a9c141d0c0b 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,7 +3,6 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() -#include "pycore_code.h" // _Py_next_func_version #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -252,11 +251,9 @@ When the function version is 0, the `CALL` bytecode is not specialized. Code object versions -------------------- -So where to code objects get their `co_version`? There is a single -static global counter, `_Py_next_func_version`. This is initialized in -the generated (!) file `Python/deepfreeze/deepfreeze.c`, to 1 plus the -number of deep-frozen function objects in that file. -(In `_bootstrap_python.c` and `freeze_module.c` it is initialized to 1.) +So where to code objects get their `co_version`? +There is a per-interpreter counter, `next_func_version`. +This is initialized to 1 when the interpreter is created. Code objects get a new `co_version` allocated from this counter upon creation. Since code objects are nominally immutable, `co_version` can diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6c388fc7033dd0..34f79191b4e8d7 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -15,8 +15,6 @@ #include "Python/frozen_modules/zipimport.h" /* End includes */ -uint32_t _Py_next_func_version = 1; - /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index f6c46fa629efba..3de6c6816c1e61 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -22,8 +22,6 @@ # include #endif -uint32_t _Py_next_func_version = 1; - /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Python/frozen.c b/Python/frozen.c index 6b977710e6e342..0fb38a11902f35 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -101,46 +101,46 @@ extern PyObject *_Py_get_frozen_only_toplevel(void); /* End extern declarations */ static const struct _frozen bootstrap_modules[] = { - {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false, GET_CODE(importlib__bootstrap)}, - {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false, GET_CODE(importlib__bootstrap_external)}, - {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false, GET_CODE(zipimport)}, + {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, + {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, + {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ - {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false, GET_CODE(abc)}, - {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false, GET_CODE(codecs)}, - {"io", _Py_M__io, (int)sizeof(_Py_M__io), false, GET_CODE(io)}, + {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false}, + {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false}, + {"io", _Py_M__io, (int)sizeof(_Py_M__io), false}, /* stdlib - startup, with site */ - {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), false, GET_CODE(_collections_abc)}, - {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), false, GET_CODE(_sitebuiltins)}, - {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), false, GET_CODE(genericpath)}, - {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), false, GET_CODE(ntpath)}, - {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false, GET_CODE(posixpath)}, - {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false, GET_CODE(posixpath)}, - {"os", _Py_M__os, (int)sizeof(_Py_M__os), false, GET_CODE(os)}, - {"site", _Py_M__site, (int)sizeof(_Py_M__site), false, GET_CODE(site)}, - {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false, GET_CODE(stat)}, + {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), false}, + {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), false}, + {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), false}, + {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), false}, + {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false}, + {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false}, + {"os", _Py_M__os, (int)sizeof(_Py_M__os), false}, + {"site", _Py_M__site, (int)sizeof(_Py_M__site), false}, + {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false}, /* runpy - run module with -m */ - {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false, GET_CODE(importlib_util)}, - {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false, GET_CODE(importlib_machinery)}, - {"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), false, GET_CODE(runpy)}, + {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false}, + {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false}, + {"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), false}, {0, 0, 0} /* stdlib sentinel */ }; static const struct _frozen test_modules[] = { - {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)}, - {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)}, - {"__phello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), true, GET_CODE(__hello__)}, - {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)}, - {"__phello__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), true, GET_CODE(__phello__)}, - {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), false, GET_CODE(__phello__)}, - {"__phello__.ham", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), true, GET_CODE(__phello___ham)}, - {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), false, GET_CODE(__phello___ham)}, - {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), false, GET_CODE(__phello___ham_eggs)}, - {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), false, GET_CODE(__phello___spam)}, - {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), false, GET_CODE(frozen_only)}, + {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false}, + {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false}, + {"__phello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), true}, + {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false}, + {"__phello__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), true}, + {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), false}, + {"__phello__.ham", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), true}, + {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), false}, + {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), false}, + {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), false}, + {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), false}, {0, 0, 0} /* test sentinel */ }; const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules; diff --git a/Python/import.c b/Python/import.c index 48090d05240188..126eb5e162a9e1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2006,7 +2006,6 @@ look_up_frozen(const char *name) struct frozen_info { PyObject *nameobj; const char *data; - PyObject *(*get_code)(void); Py_ssize_t size; bool is_package; bool is_alias; @@ -2040,7 +2039,6 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) if (info != NULL) { info->nameobj = nameobj; // borrowed info->data = (const char *)p->code; - info->get_code = p->get_code; info->size = p->size; info->is_package = p->is_package; if (p->size < 0) { @@ -2052,10 +2050,6 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, &info->origname); } - if (p->code == NULL && p->size == 0 && p->get_code != NULL) { - /* It is only deepfrozen. */ - return FROZEN_OKAY; - } if (p->code == NULL) { /* It is frozen but marked as un-importable. */ return FROZEN_EXCLUDED; @@ -2070,11 +2064,6 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) static PyObject * unmarshal_frozen_code(PyInterpreterState *interp, struct frozen_info *info) { - if (info->get_code && _Py_IsMainInterpreter(interp)) { - PyObject *code = info->get_code(); - assert(code != NULL); - return code; - } PyObject *co = PyMarshal_ReadObjectFromString(info->data, info->size); if (co == NULL) { /* Does not contain executable code. */ @@ -3567,7 +3556,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, if (info.nameobj == NULL) { info.nameobj = name; } - if (info.size == 0 && info.get_code == NULL) { + if (info.size == 0) { /* Does not contain executable code. */ set_frozen_error(FROZEN_INVALID, name); return NULL; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 92eef6d50712bb..480001538540bb 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -832,11 +832,6 @@ pycore_interp_init(PyThreadState *tstate) if (_PyStatus_EXCEPTION(status)) { return status; } - // Intern strings in deep-frozen modules first so that others - // can use it instead of creating a heap allocated string. - if (_Py_Deepfreeze_Init() < 0) { - return _PyStatus_ERR("failed to initialize deep-frozen modules"); - } status = pycore_init_types(interp); if (_PyStatus_EXCEPTION(status)) { @@ -1743,7 +1738,6 @@ finalize_interp_clear(PyThreadState *tstate) _Py_HashRandomization_Fini(); _PyArg_Fini(); _Py_ClearFileSystemEncoding(); - _Py_Deepfreeze_Fini(); _PyPerfTrampoline_Fini(); } diff --git a/Python/pystate.c b/Python/pystate.c index b2b9b9f8776c33..ed14f82688f401 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -697,6 +697,7 @@ init_interpreter(PyInterpreterState *interp, interp->optimizer = &_PyOptimizer_Default; interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; + interp->next_func_version = 1; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 12200979fa4bc7..a07f4d9786ea65 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -467,15 +467,14 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] -def regen_frozen(modules, frozen_modules: bool): +def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) - if frozen_modules: - for src in _iter_sources(modules): - # Adding a comment to separate sections here doesn't add much, - # so we don't. - header = relpath_for_posix_display(src.frozenfile, parentdir) - headerlines.append(f'#include "{header}"') + for src in _iter_sources(modules): + # Adding a comment to separate sections here doesn't add much, + # so we don't. + header = relpath_for_posix_display(src.frozenfile, parentdir) + headerlines.append(f'#include "{header}"') externlines = [] bootstraplines = [] @@ -504,14 +503,9 @@ def regen_frozen(modules, frozen_modules: bool): get_code_name = "_Py_get_%s_toplevel" % code_name externlines.append("extern PyObject *%s(void);" % get_code_name) - symbol = mod.symbol pkg = 'true' if mod.ispkg else 'false' - if not frozen_modules: - line = ('{"%s", NULL, 0, %s, GET_CODE(%s)},' - ) % (mod.name, pkg, code_name) - else: - line = ('{"%s", %s, (int)sizeof(%s), %s, GET_CODE(%s)},' - ) % (mod.name, symbol, symbol, pkg, code_name) + size = f"(int)sizeof({mod.symbol})" + line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},' lines.append(line) if mod.isalias: @@ -718,20 +712,14 @@ def regen_pcbuild(modules): ####################################### # the script -parser = argparse.ArgumentParser() -parser.add_argument("--frozen-modules", action="store_true", - help="Use both frozen and deepfrozen modules. (default: uses only deepfrozen modules)") - def main(): - args = parser.parse_args() - frozen_modules: bool = args.frozen_modules # Expand the raw specs, preserving order. modules = list(parse_frozen_specs()) # Regen build-related files. regen_makefile(modules) regen_pcbuild(modules) - regen_frozen(modules, frozen_modules) + regen_frozen(modules) if __name__ == '__main__': From b0edf3b98e4b3e68a13776e034b9dd86ad7e529d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 11:48:28 +0200 Subject: [PATCH 562/598] GH-91079: Rename C_RECURSION_LIMIT to Py_C_RECURSION_LIMIT (#108507) Symbols of the C API should be prefixed by "Py_" to avoid conflict with existing names in 3rd party C extensions on "#include ". test.pythoninfo now logs Py_C_RECURSION_LIMIT constant and other _testcapi and _testinternalcapi constants. --- Include/cpython/pystate.h | 19 +++++++++---------- Lib/test/list_tests.py | 4 ++-- Lib/test/mapping_tests.py | 4 ++-- Lib/test/pythoninfo.py | 23 +++++++++++++++++++++++ Lib/test/support/__init__.py | 4 ++-- Lib/test/test_compile.py | 12 ++++++------ Lib/test/test_dict.py | 4 ++-- Lib/test/test_dictviews.py | 4 ++-- Lib/test/test_exception_group.py | 4 ++-- Modules/_testcapimodule.c | 1 + Modules/_testinternalcapi.c | 5 +++++ Parser/asdl_c.py | 4 ++-- Python/Python-ast.c | 4 ++-- Python/ast.c | 4 ++-- Python/ast_opt.c | 4 ++-- Python/pystate.c | 2 +- Python/symtable.c | 4 ++-- 17 files changed, 67 insertions(+), 39 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index fc5f58db86dbe8..e1a15cddd3d723 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -194,18 +194,17 @@ struct _ts { }; -/* WASI has limited call stack. Python's recursion limit depends on code - layout, optimization, and WASI runtime. Wasmtime can handle about 700 - recursions, sometimes less. 500 is a more conservative limit. */ -#ifndef C_RECURSION_LIMIT -# ifdef __wasi__ -# define C_RECURSION_LIMIT 500 -# else - // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 1500 -# endif +#ifdef __wasi__ + // WASI has limited call stack. Python's recursion limit depends on code + // layout, optimization, and WASI runtime. Wasmtime can handle about 700 + // recursions, sometimes less. 500 is a more conservative limit. +# define Py_C_RECURSION_LIMIT 500 +#else + // This value is duplicated in Lib/test/support/__init__.py +# define Py_C_RECURSION_LIMIT 1500 #endif + /* other API */ /* Similar to PyThreadState_Get(), but don't issue a fatal error diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index b1ef332522d2ce..d9ab21d4941cdb 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT +from test.support import ALWAYS_EQ, NEVER_EQ, Py_C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 5492bbf86d1f87..b3e4192e65d957 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,7 +2,7 @@ import unittest import collections import sys -from test.support import C_RECURSION_LIMIT +from test.support import Py_C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -625,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index c628833478044e..b25def78e42be4 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -665,6 +665,22 @@ def collect_decimal(info_add): def collect_testcapi(info_add): + try: + import _testcapi + except ImportError: + return + + for name in ( + 'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix + 'PY_SSIZE_T_MAX', + 'Py_C_RECURSION_LIMIT', + 'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform + 'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform + ): + copy_attr(info_add, f'_testcapi.{name}', _testcapi, name) + + +def collect_testinternalcapi(info_add): try: import _testinternalcapi except ImportError: @@ -672,6 +688,12 @@ def collect_testcapi(info_add): call_func(info_add, 'pymem.allocator', _testinternalcapi, 'pymem_getallocatorsname') + for name in ( + 'SIZEOF_PYGC_HEAD', + 'SIZEOF_PYOBJECT', + ): + copy_attr(info_add, f'_testinternalcapi.{name}', _testinternalcapi, name) + def collect_resource(info_add): try: @@ -907,6 +929,7 @@ def collect_info(info): collect_sys, collect_sysconfig, collect_testcapi, + collect_testinternalcapi, collect_time, collect_tkinter, collect_windows, diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 38ad965e155302..84b74ee2c298e9 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -60,7 +60,7 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "Py_C_RECURSION_LIMIT", "skip_on_s390x", ] @@ -2531,7 +2531,7 @@ def adjust_int_max_str_digits(max_digits): EXCEEDS_RECURSION_LIMIT = 5000 # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +Py_C_RECURSION_LIMIT = 1500 #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index de513daf825d81..28b2c4686bbc89 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,7 +11,7 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization, C_RECURSION_LIMIT) + requires_specialization, Py_C_RECURSION_LIMIT) from test.support.os_helper import FakePath class TestSpecifics(unittest.TestCase): @@ -111,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = int(C_RECURSION_LIMIT * 0.9) + repeat = int(Py_C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -557,12 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is C_RECURSION_LIMIT * 2 + # Expected limit is Py_C_RECURSION_LIMIT * 2 # Duplicating the limit here is a little ugly. # Perhaps it should be exposed somewhere... - fail_depth = C_RECURSION_LIMIT * 2 + 1 - crash_depth = C_RECURSION_LIMIT * 100 - success_depth = int(C_RECURSION_LIMIT * 1.8) + fail_depth = Py_C_RECURSION_LIMIT * 2 + 1 + crash_depth = Py_C_RECURSION_LIMIT * 100 + success_depth = int(Py_C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index eab64b4f9106c1..620d0ca4f4c2da 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, C_RECURSION_LIMIT +from test.support import import_helper, Py_C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 2bd9d6eef8cfc6..34918585513846 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,7 +3,7 @@ import pickle import sys import unittest -from test.support import C_RECURSION_LIMIT +from test.support import Py_C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -280,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(C_RECURSION_LIMIT//2 + 100): + for i in range(Py_C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index a02d54da35e948..20122679223843 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import C_RECURSION_LIMIT +from test.support import Py_C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(C_RECURSION_LIMIT + 1): + for i in range(Py_C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9bd963baf066a4..85d8401435e1b5 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3922,6 +3922,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); + PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index b6792e38fa98c2..922672d1a9f915 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1552,6 +1552,11 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "SIZEOF_PYOBJECT", + PyLong_FromSsize_t(sizeof(PyObject))) < 0) { + return 1; + } + if (PyModule_Add(module, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))) < 0) { return 1; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ca259c8cd1f3ba..f61099b97055ad 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1401,8 +1401,8 @@ class PartingShots(StaticVisitor): if (!tstate) { return 0; } - state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state->recursion_depth = starting_recursion_depth; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 77b23f7c5edf23..a197d44868b5aa 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13081,8 +13081,8 @@ PyObject* PyAST_mod2obj(mod_ty t) if (!tstate) { return 0; } - state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state->recursion_depth = starting_recursion_depth; diff --git a/Python/ast.c b/Python/ast.c index 74c97f948d15e6..21cb38f8cbfb65 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1046,10 +1046,10 @@ _PyAST_Validate(mod_ty mod) return 0; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 82e7559e5b629a..41f48eba08afc4 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1130,10 +1130,10 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) return 0; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); diff --git a/Python/pystate.c b/Python/pystate.c index ed14f82688f401..89275fd7e025ca 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1334,7 +1334,7 @@ init_threadstate(PyThreadState *tstate, tstate->py_recursion_limit = interp->ceval.recursion_limit, tstate->py_recursion_remaining = interp->ceval.recursion_limit, - tstate->c_recursion_remaining = C_RECURSION_LIMIT; + tstate->c_recursion_remaining = Py_C_RECURSION_LIMIT; tstate->exc_info = &tstate->exc_state; diff --git a/Python/symtable.c b/Python/symtable.c index f157d4c170314a..217e6f59a61484 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -312,10 +312,10 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) return NULL; } /* Be careful here to prevent overflow. */ - int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; + int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { From f63d37877ad166041489a968233b57540f8456e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 11:50:46 +0200 Subject: [PATCH 563/598] gh-104690: thread_run() checks for tstate dangling pointer (#109056) thread_run() of _threadmodule.c now calls _PyThreadState_CheckConsistency() to check if tstate is a dangling pointer when Python is built in debug mode. Rename ceval_gil.c is_tstate_valid() to _PyThreadState_CheckConsistency() to reuse it in _threadmodule.c. --- Include/internal/pycore_pystate.h | 4 ++++ Modules/_threadmodule.c | 7 +++++-- Python/ceval_gil.c | 26 ++++++++------------------ Python/pystate.c | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index a30036aeb57e05..9c0e42e7bad06c 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -67,6 +67,10 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) extern _Py_thread_local PyThreadState *_Py_tss_tstate; #endif +#ifndef NDEBUG +extern int _PyThreadState_CheckConsistency(PyThreadState *tstate); +#endif + // Export for most shared extensions, used via _PyThreadState_GET() static // inline function. PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 49f34fcb9feb70..05bb49756c9303 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1074,9 +1074,12 @@ static void thread_run(void *boot_raw) { struct bootstate *boot = (struct bootstate *) boot_raw; - PyThreadState *tstate; + PyThreadState *tstate = boot->tstate; + + // gh-104690: If Python is being finalized and PyInterpreterState_Delete() + // was called, tstate becomes a dangling pointer. + assert(_PyThreadState_CheckConsistency(tstate)); - tstate = boot->tstate; _PyThreadState_Bind(tstate); PyEval_AcquireThread(tstate); tstate->interp->threads.count++; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index e53ffa76b1164b..cef5317b46bf8e 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -163,16 +163,6 @@ UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp) COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); } -#ifndef NDEBUG -/* Ensure that tstate is valid */ -static int -is_tstate_valid(PyThreadState *tstate) -{ - assert(!_PyMem_IsPtrFreed(tstate)); - assert(!_PyMem_IsPtrFreed(tstate->interp)); - return 1; -} -#endif /* * Implementation of the Global Interpreter Lock (GIL). @@ -325,7 +315,7 @@ drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); RESET_GIL_DROP_REQUEST(tstate->interp); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take @@ -386,7 +376,7 @@ take_gil(PyThreadState *tstate) PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _ceval_state *ceval = &interp->ceval; struct _gil_runtime_state *gil = ceval->gil; @@ -427,7 +417,7 @@ take_gil(PyThreadState *tstate) } PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); SET_GIL_DROP_REQUEST(interp); drop_requested = 1; @@ -466,7 +456,7 @@ take_gil(PyThreadState *tstate) drop_gil(ceval, tstate); PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { RESET_GIL_DROP_REQUEST(interp); @@ -679,7 +669,7 @@ PyEval_AcquireThread(PyThreadState *tstate) void PyEval_ReleaseThread(PyThreadState *tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); if (new_tstate != tstate) { @@ -877,7 +867,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) static int handle_signals(PyThreadState *tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } @@ -983,7 +973,7 @@ void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (make_pending_calls(tstate->interp) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); @@ -1024,7 +1014,7 @@ Py_MakePendingCalls(void) assert(PyGILState_Check()); PyThreadState *tstate = _PyThreadState_GET(); - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); /* Only execute pending calls on the main thread. */ if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(tstate->interp)) { diff --git a/Python/pystate.c b/Python/pystate.c index 89275fd7e025ca..09c3538ad7b872 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2890,6 +2890,24 @@ _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame * frame) } +#ifndef NDEBUG +// Check that a Python thread state valid. In practice, this function is used +// on a Python debug build to check if 'tstate' is a dangling pointer, if the +// PyThreadState memory has been freed. +// +// Usage: +// +// assert(_PyThreadState_CheckConsistency(tstate)); +int +_PyThreadState_CheckConsistency(PyThreadState *tstate) +{ + assert(!_PyMem_IsPtrFreed(tstate)); + assert(!_PyMem_IsPtrFreed(tstate->interp)); + return 1; +} +#endif + + #ifdef __cplusplus } #endif From aa51182320f3c391195eb7d5bd970867e63bd978 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 8 Sep 2023 09:30:28 -0600 Subject: [PATCH 564/598] gh-109140: Rename duplicated tests in `test_binascii` (#109141) --- Lib/test/test_binascii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index eb831b1a06fcb8..3d3e0746e9bfdf 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -233,7 +233,7 @@ def test_uu(self): binary=hypothesis.strategies.binary(), backtick=hypothesis.strategies.booleans(), ) - def test_hex_roundtrip(self, binary, backtick): + def test_b2a_roundtrip(self, binary, backtick): converted = binascii.b2a_uu(self.type2test(binary), backtick=backtick) restored = binascii.a2b_uu(self.type2test(converted)) self.assertConversion(binary, converted, restored, backtick=backtick) From 6275c67ea68645e5b296a80ea63b90707a0be792 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 8 Sep 2023 17:18:35 +0100 Subject: [PATCH 565/598] gh-106922: Fix error location for constructs with spaces and parentheses (#108959) --- Lib/test/test_traceback.py | 36 +++++++++++++++++++ Lib/traceback.py | 16 +++++++-- ...-09-05-20-52-17.gh-issue-108959.6z45Sy.rst | 2 ++ Python/traceback.c | 17 +++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 316ade2171e94f..be81082bb19472 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -596,6 +596,24 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = 1 + b = "" + return ( a ) + b + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return ( a ) + b\n' + ' ~~~~~~~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript(self): def f_with_subscript(): some_dict = {'x': {'y': None}} @@ -630,6 +648,24 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = [] + b = c = 1 + return b [ a ] + c + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return b [ a ] + c\n' + ' ~~~~~~^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_traceback_specialization_with_syntax_error(self): bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") diff --git a/Lib/traceback.py b/Lib/traceback.py index 354754b9560a19..67941ff45988c2 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -608,11 +608,21 @@ def _extract_caret_anchors_from_line_segment(segment): and not operator_str[operator_offset + 1].isspace() ): right_anchor += 1 + + while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch in ")#"): + left_anchor += 1 + right_anchor += 1 return _Anchors(normalize(left_anchor), normalize(right_anchor)) case ast.Subscript(): - subscript_start = normalize(expr.value.end_col_offset) - subscript_end = normalize(expr.slice.end_col_offset + 1) - return _Anchors(subscript_start, subscript_end) + left_anchor = normalize(expr.value.end_col_offset) + right_anchor = normalize(expr.slice.end_col_offset + 1) + while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch != "["): + left_anchor += 1 + while right_anchor < len(segment) and ((ch := segment[right_anchor]).isspace() or ch != "]"): + right_anchor += 1 + if right_anchor < len(segment): + right_anchor += 1 + return _Anchors(left_anchor, right_anchor) return None diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst new file mode 100644 index 00000000000000..792bbc454f2b27 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-05-20-52-17.gh-issue-108959.6z45Sy.rst @@ -0,0 +1,2 @@ +Fix caret placement for error locations for subscript and binary operations +that involve non-semantic parentheses and spaces. Patch by Pablo Galindo diff --git a/Python/traceback.c b/Python/traceback.c index 2fcfa7ca56c140..a75b7833af4e05 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -616,6 +616,11 @@ extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *lef ++*right_anchor; } + // Keep going if the current char is not ')' + if (i+1 < right->col_offset && (segment_str[i] == ')')) { + continue; + } + // Set the error characters *primary_error_char = "~"; *secondary_error_char = "^"; @@ -626,6 +631,18 @@ extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *lef case Subscript_kind: { *left_anchor = expr->v.Subscript.value->end_col_offset; *right_anchor = expr->v.Subscript.slice->end_col_offset + 1; + Py_ssize_t str_len = strlen(segment_str); + + // Move right_anchor and left_anchor forward to the first non-whitespace character that is not ']' and '[' + while (*left_anchor < str_len && (IS_WHITESPACE(segment_str[*left_anchor]) || segment_str[*left_anchor] != '[')) { + ++*left_anchor; + } + while (*right_anchor < str_len && (IS_WHITESPACE(segment_str[*right_anchor]) || segment_str[*right_anchor] != ']')) { + ++*right_anchor; + } + if (*right_anchor < str_len){ + *right_anchor += 1; + } // Set the error characters *primary_error_char = "~"; From 52beebc856fedf507ac0eb9e45c2e2c9fed1e5b8 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 8 Sep 2023 12:23:58 -0400 Subject: [PATCH 566/598] gh-109136: Fix summarize_stats.py tool (#109137) --- Tools/scripts/summarize_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 2d198506fb5c6c..55b67643977a00 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -430,7 +430,7 @@ def emit_comparative_specialization_overview(base_opcode_stats, base_total, head ) def get_stats_defines(): - stats_path = os.path.join(os.path.dirname(__file__), "../../Include/pystats.h") + stats_path = os.path.join(os.path.dirname(__file__), "../../Include/cpython/pystats.h") with open(stats_path) as stats_src: defines = parse_kinds(stats_src, prefix="EVAL_CALL") return defines From ccd48623d4860e730a16f3f252d67bfea8c1e905 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 8 Sep 2023 21:57:58 +0530 Subject: [PATCH 567/598] GH-109067: fix randomly failing `test_async_gen_asyncio_gc_aclose_09` test (#109142) Use `asyncio.sleep(0)` instead of short sleeps. --- Lib/test/test_asyncgen.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4f00558770dafd..a49630112af510 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1058,8 +1058,7 @@ async def gen(): while True: yield 1 finally: - await asyncio.sleep(0.01) - await asyncio.sleep(0.01) + await asyncio.sleep(0) DONE = 1 async def run(): @@ -1069,7 +1068,10 @@ async def run(): del g gc_collect() # For PyPy or other GCs. - await asyncio.sleep(0.1) + # Starts running the aclose task + await asyncio.sleep(0) + # For asyncio.sleep(0) in finally block + await asyncio.sleep(0) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) From 501f2dc527010a4fe17026b3f352d72b01228b6f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 8 Sep 2023 17:54:45 +0100 Subject: [PATCH 568/598] GH-108614: Unbreak emscripten build (GH-109132) --- Include/internal/pycore_emscripten_signal.h | 1 + Python/bytecodes.c | 4 ++-- Python/emscripten_signal.c | 8 ++++---- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_emscripten_signal.h b/Include/internal/pycore_emscripten_signal.h index d1bcb9a92c7726..754193e21dec5a 100644 --- a/Include/internal/pycore_emscripten_signal.h +++ b/Include/internal/pycore_emscripten_signal.h @@ -18,6 +18,7 @@ _Py_CheckEmscriptenSignalsPeriodically(void); #define _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY() _Py_CheckEmscriptenSignalsPeriodically() extern int Py_EMSCRIPTEN_SIGNAL_HANDLING; +extern int _Py_emscripten_signal_clock; #else diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8820b52774671b..21069204cf8693 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -154,8 +154,8 @@ dummy_func( inst(RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(emscripten_signal_clock == 0, RESUME); - emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version diff --git a/Python/emscripten_signal.c b/Python/emscripten_signal.c index 1a196385b8ce34..561b5b73cd6b70 100644 --- a/Python/emscripten_signal.c +++ b/Python/emscripten_signal.c @@ -39,16 +39,16 @@ _Py_CheckEmscriptenSignals(void) } #define PY_EMSCRIPTEN_SIGNAL_INTERVAL 50 -static int emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; +int _Py_emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; void _Py_CheckEmscriptenSignalsPeriodically(void) { - if (emscripten_signal_clock == 0) { - emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; + if (_Py_emscripten_signal_clock == 0) { + _Py_emscripten_signal_clock = PY_EMSCRIPTEN_SIGNAL_INTERVAL; _Py_CheckEmscriptenSignals(); } else if (Py_EMSCRIPTEN_SIGNAL_HANDLING) { - emscripten_signal_clock--; + _Py_emscripten_signal_clock--; } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f4c526a5a8c0a2..fa7cb88e4a1e1f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9,8 +9,8 @@ case RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - DEOPT_IF(emscripten_signal_clock == 0, RESUME); - emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 84f83db128ea50..136b36c8260cb5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -28,8 +28,8 @@ TARGET(RESUME_CHECK) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(emscripten_signal_clock == 0, RESUME); - emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version From 87a7faf6b68c8076e640a9a1347a255f132d8382 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 8 Sep 2023 19:57:41 +0300 Subject: [PATCH 569/598] Check the result of PySet_Contains() for error in Python/symtable.c (GH-109146) --- Python/symtable.c | 72 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 217e6f59a61484..7eef6f7231c035 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -523,6 +523,7 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { + int contains; if (flags & DEF_GLOBAL) { if (flags & DEF_NONLOCAL) { PyErr_Format(PyExc_SyntaxError, @@ -543,14 +544,22 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, "nonlocal declaration not allowed at module level"); return error_at_directive(ste, name); } - if (!PySet_Contains(bound, name)) { + contains = PySet_Contains(bound, name); + if (contains < 0) { + return 0; + } + if (!contains) { PyErr_Format(PyExc_SyntaxError, "no binding for nonlocal '%U' found", name); return error_at_directive(ste, name); } - if (PySet_Contains(type_params, name)) { + contains = PySet_Contains(type_params, name); + if (contains < 0) { + return 0; + } + if (contains) { PyErr_Format(PyExc_SyntaxError, "nonlocal binding not allowed for type parameter '%U'", name); @@ -599,17 +608,29 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, Note that having a non-NULL bound implies that the block is nested. */ - if (bound && PySet_Contains(bound, name)) { - SET_SCOPE(scopes, name, FREE); - ste->ste_free = 1; - return PySet_Add(free, name) >= 0; + if (bound) { + contains = PySet_Contains(bound, name); + if (contains < 0) { + return 0; + } + if (contains) { + SET_SCOPE(scopes, name, FREE); + ste->ste_free = 1; + return PySet_Add(free, name) >= 0; + } } /* If a parent has a global statement, then call it global explicit? It could also be global implicit. */ - if (global && PySet_Contains(global, name)) { - SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); - return 1; + if (global) { + contains = PySet_Contains(global, name); + if (contains < 0) { + return 0; + } + if (contains) { + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); + return 1; + } } if (ste->ste_nested) ste->ste_free = 1; @@ -712,8 +733,19 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) scope = PyLong_AS_LONG(v); if (scope != LOCAL) continue; - if (!PySet_Contains(free, name) && !PySet_Contains(inlined_cells, name)) - continue; + int contains = PySet_Contains(free, name); + if (contains < 0) { + goto error; + } + if (!contains) { + contains = PySet_Contains(inlined_cells, name); + if (contains < 0) { + goto error; + } + if (!contains) { + continue; + } + } /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. @@ -764,7 +796,11 @@ update_symbols(PyObject *symbols, PyObject *scopes, long scope, flags; assert(PyLong_Check(v)); flags = PyLong_AS_LONG(v); - if (PySet_Contains(inlined_cells, name)) { + int contains = PySet_Contains(inlined_cells, name); + if (contains < 0) { + return 0; + } + if (contains) { flags |= DEF_COMP_CELL; } v_scope = PyDict_GetItemWithError(scopes, name); @@ -822,9 +858,15 @@ update_symbols(PyObject *symbols, PyObject *scopes, goto error; } /* Handle global symbol */ - if (bound && !PySet_Contains(bound, name)) { - Py_DECREF(name); - continue; /* it's a global */ + if (bound) { + int contains = PySet_Contains(bound, name); + if (contains < 0) { + goto error; + } + if (!contains) { + Py_DECREF(name); + continue; /* it's a global */ + } } /* Propagate new free symbol up the lexical stack */ if (PyDict_SetItem(symbols, name, v_free) < 0) { From 5bda2f637e1cfbca45a83aa6e22db25498064b27 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 8 Sep 2023 18:00:23 +0100 Subject: [PATCH 570/598] gh-109114: Relax the check for invalid lambdas inside f-strings to avoid false positives (#109121) --- Grammar/python.gram | 2 +- Lib/test/test_fstring.py | 4 + ...-09-08-01-50-41.gh-issue-109114.adqgtb.rst | 3 + Parser/parser.c | 2557 ++++++++--------- 4 files changed, 1257 insertions(+), 1309 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index e7c817856d514b..ae998f98076a0a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1170,7 +1170,7 @@ invalid_expression: _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } - | a='lambda' [lambda_params] b=':' &(FSTRING_MIDDLE | fstring_replacement_field) { + | a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") } invalid_named_expression(memo): diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 16f01973f99f3e..4f05a149a901b2 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1027,6 +1027,10 @@ def test_lambda(self): "f'{lambda x:}'", "f'{lambda :}'", ]) + # Ensure the detection of invalid lambdas doesn't trigger detection + # for valid lambdas in the second error pass + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + compile("lambda name_3=f'{name_4}': {name_3}\n1 $ 1", "", "exec") # but don't emit the paren warning in general cases with self.assertRaisesRegex(SyntaxError, "f-string: expecting a valid expression after '{'"): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst new file mode 100644 index 00000000000000..3d95dd5d29450c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-08-01-50-41.gh-issue-109114.adqgtb.rst @@ -0,0 +1,3 @@ +Relax the detection of the error message for invalid lambdas inside +f-strings to not search for arbitrary replacement fields to avoid false +positives. Patch by Pablo Galindo diff --git a/Parser/parser.c b/Parser/parser.c index 95cbef4a64a10f..e5847e7d1bd920 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -483,65 +483,65 @@ static char *soft_keywords[] = { #define _tmp_157_type 1400 #define _tmp_158_type 1401 #define _tmp_159_type 1402 -#define _tmp_160_type 1403 +#define _loop0_160_type 1403 #define _loop0_161_type 1404 #define _loop0_162_type 1405 -#define _loop0_163_type 1406 +#define _tmp_163_type 1406 #define _tmp_164_type 1407 #define _tmp_165_type 1408 #define _tmp_166_type 1409 #define _tmp_167_type 1410 -#define _tmp_168_type 1411 +#define _loop0_168_type 1411 #define _loop0_169_type 1412 #define _loop0_170_type 1413 -#define _loop0_171_type 1414 -#define _loop1_172_type 1415 -#define _tmp_173_type 1416 -#define _loop0_174_type 1417 -#define _tmp_175_type 1418 -#define _loop0_176_type 1419 -#define _loop1_177_type 1420 +#define _loop1_171_type 1414 +#define _tmp_172_type 1415 +#define _loop0_173_type 1416 +#define _tmp_174_type 1417 +#define _loop0_175_type 1418 +#define _loop1_176_type 1419 +#define _tmp_177_type 1420 #define _tmp_178_type 1421 #define _tmp_179_type 1422 -#define _tmp_180_type 1423 -#define _loop0_181_type 1424 +#define _loop0_180_type 1423 +#define _tmp_181_type 1424 #define _tmp_182_type 1425 -#define _tmp_183_type 1426 -#define _loop1_184_type 1427 -#define _tmp_185_type 1428 +#define _loop1_183_type 1426 +#define _tmp_184_type 1427 +#define _loop0_185_type 1428 #define _loop0_186_type 1429 #define _loop0_187_type 1430 -#define _loop0_188_type 1431 -#define _loop0_190_type 1432 -#define _gather_189_type 1433 -#define _tmp_191_type 1434 -#define _loop0_192_type 1435 -#define _tmp_193_type 1436 -#define _loop0_194_type 1437 +#define _loop0_189_type 1431 +#define _gather_188_type 1432 +#define _tmp_190_type 1433 +#define _loop0_191_type 1434 +#define _tmp_192_type 1435 +#define _loop0_193_type 1436 +#define _loop1_194_type 1437 #define _loop1_195_type 1438 -#define _loop1_196_type 1439 +#define _tmp_196_type 1439 #define _tmp_197_type 1440 -#define _tmp_198_type 1441 -#define _loop0_199_type 1442 +#define _loop0_198_type 1441 +#define _tmp_199_type 1442 #define _tmp_200_type 1443 #define _tmp_201_type 1444 -#define _tmp_202_type 1445 -#define _loop0_204_type 1446 -#define _gather_203_type 1447 -#define _loop0_206_type 1448 -#define _gather_205_type 1449 -#define _loop0_208_type 1450 -#define _gather_207_type 1451 -#define _loop0_210_type 1452 -#define _gather_209_type 1453 -#define _loop0_212_type 1454 -#define _gather_211_type 1455 -#define _tmp_213_type 1456 -#define _loop0_214_type 1457 -#define _loop1_215_type 1458 -#define _tmp_216_type 1459 -#define _loop0_217_type 1460 -#define _loop1_218_type 1461 +#define _loop0_203_type 1445 +#define _gather_202_type 1446 +#define _loop0_205_type 1447 +#define _gather_204_type 1448 +#define _loop0_207_type 1449 +#define _gather_206_type 1450 +#define _loop0_209_type 1451 +#define _gather_208_type 1452 +#define _loop0_211_type 1453 +#define _gather_210_type 1454 +#define _tmp_212_type 1455 +#define _loop0_213_type 1456 +#define _loop1_214_type 1457 +#define _tmp_215_type 1458 +#define _loop0_216_type 1459 +#define _loop1_217_type 1460 +#define _tmp_218_type 1461 #define _tmp_219_type 1462 #define _tmp_220_type 1463 #define _tmp_221_type 1464 @@ -551,9 +551,9 @@ static char *soft_keywords[] = { #define _tmp_225_type 1468 #define _tmp_226_type 1469 #define _tmp_227_type 1470 -#define _tmp_228_type 1471 -#define _loop0_230_type 1472 -#define _gather_229_type 1473 +#define _loop0_229_type 1471 +#define _gather_228_type 1472 +#define _tmp_230_type 1473 #define _tmp_231_type 1474 #define _tmp_232_type 1475 #define _tmp_233_type 1476 @@ -566,8 +566,8 @@ static char *soft_keywords[] = { #define _tmp_240_type 1483 #define _tmp_241_type 1484 #define _tmp_242_type 1485 -#define _tmp_243_type 1486 -#define _loop0_244_type 1487 +#define _loop0_243_type 1486 +#define _tmp_244_type 1487 #define _tmp_245_type 1488 #define _tmp_246_type 1489 #define _tmp_247_type 1490 @@ -599,7 +599,6 @@ static char *soft_keywords[] = { #define _tmp_273_type 1516 #define _tmp_274_type 1517 #define _tmp_275_type 1518 -#define _tmp_276_type 1519 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -1004,65 +1003,65 @@ static void *_tmp_156_rule(Parser *p); static void *_tmp_157_rule(Parser *p); static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); -static void *_tmp_160_rule(Parser *p); +static asdl_seq *_loop0_160_rule(Parser *p); static asdl_seq *_loop0_161_rule(Parser *p); static asdl_seq *_loop0_162_rule(Parser *p); -static asdl_seq *_loop0_163_rule(Parser *p); +static void *_tmp_163_rule(Parser *p); static void *_tmp_164_rule(Parser *p); static void *_tmp_165_rule(Parser *p); static void *_tmp_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); -static void *_tmp_168_rule(Parser *p); +static asdl_seq *_loop0_168_rule(Parser *p); static asdl_seq *_loop0_169_rule(Parser *p); static asdl_seq *_loop0_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); -static asdl_seq *_loop1_172_rule(Parser *p); -static void *_tmp_173_rule(Parser *p); -static asdl_seq *_loop0_174_rule(Parser *p); -static void *_tmp_175_rule(Parser *p); -static asdl_seq *_loop0_176_rule(Parser *p); -static asdl_seq *_loop1_177_rule(Parser *p); +static asdl_seq *_loop1_171_rule(Parser *p); +static void *_tmp_172_rule(Parser *p); +static asdl_seq *_loop0_173_rule(Parser *p); +static void *_tmp_174_rule(Parser *p); +static asdl_seq *_loop0_175_rule(Parser *p); +static asdl_seq *_loop1_176_rule(Parser *p); +static void *_tmp_177_rule(Parser *p); static void *_tmp_178_rule(Parser *p); static void *_tmp_179_rule(Parser *p); -static void *_tmp_180_rule(Parser *p); -static asdl_seq *_loop0_181_rule(Parser *p); +static asdl_seq *_loop0_180_rule(Parser *p); +static void *_tmp_181_rule(Parser *p); static void *_tmp_182_rule(Parser *p); -static void *_tmp_183_rule(Parser *p); -static asdl_seq *_loop1_184_rule(Parser *p); -static void *_tmp_185_rule(Parser *p); +static asdl_seq *_loop1_183_rule(Parser *p); +static void *_tmp_184_rule(Parser *p); +static asdl_seq *_loop0_185_rule(Parser *p); static asdl_seq *_loop0_186_rule(Parser *p); static asdl_seq *_loop0_187_rule(Parser *p); -static asdl_seq *_loop0_188_rule(Parser *p); -static asdl_seq *_loop0_190_rule(Parser *p); -static asdl_seq *_gather_189_rule(Parser *p); -static void *_tmp_191_rule(Parser *p); -static asdl_seq *_loop0_192_rule(Parser *p); -static void *_tmp_193_rule(Parser *p); -static asdl_seq *_loop0_194_rule(Parser *p); +static asdl_seq *_loop0_189_rule(Parser *p); +static asdl_seq *_gather_188_rule(Parser *p); +static void *_tmp_190_rule(Parser *p); +static asdl_seq *_loop0_191_rule(Parser *p); +static void *_tmp_192_rule(Parser *p); +static asdl_seq *_loop0_193_rule(Parser *p); +static asdl_seq *_loop1_194_rule(Parser *p); static asdl_seq *_loop1_195_rule(Parser *p); -static asdl_seq *_loop1_196_rule(Parser *p); +static void *_tmp_196_rule(Parser *p); static void *_tmp_197_rule(Parser *p); -static void *_tmp_198_rule(Parser *p); -static asdl_seq *_loop0_199_rule(Parser *p); +static asdl_seq *_loop0_198_rule(Parser *p); +static void *_tmp_199_rule(Parser *p); static void *_tmp_200_rule(Parser *p); static void *_tmp_201_rule(Parser *p); -static void *_tmp_202_rule(Parser *p); -static asdl_seq *_loop0_204_rule(Parser *p); -static asdl_seq *_gather_203_rule(Parser *p); -static asdl_seq *_loop0_206_rule(Parser *p); -static asdl_seq *_gather_205_rule(Parser *p); -static asdl_seq *_loop0_208_rule(Parser *p); -static asdl_seq *_gather_207_rule(Parser *p); -static asdl_seq *_loop0_210_rule(Parser *p); -static asdl_seq *_gather_209_rule(Parser *p); -static asdl_seq *_loop0_212_rule(Parser *p); -static asdl_seq *_gather_211_rule(Parser *p); -static void *_tmp_213_rule(Parser *p); -static asdl_seq *_loop0_214_rule(Parser *p); -static asdl_seq *_loop1_215_rule(Parser *p); -static void *_tmp_216_rule(Parser *p); -static asdl_seq *_loop0_217_rule(Parser *p); -static asdl_seq *_loop1_218_rule(Parser *p); +static asdl_seq *_loop0_203_rule(Parser *p); +static asdl_seq *_gather_202_rule(Parser *p); +static asdl_seq *_loop0_205_rule(Parser *p); +static asdl_seq *_gather_204_rule(Parser *p); +static asdl_seq *_loop0_207_rule(Parser *p); +static asdl_seq *_gather_206_rule(Parser *p); +static asdl_seq *_loop0_209_rule(Parser *p); +static asdl_seq *_gather_208_rule(Parser *p); +static asdl_seq *_loop0_211_rule(Parser *p); +static asdl_seq *_gather_210_rule(Parser *p); +static void *_tmp_212_rule(Parser *p); +static asdl_seq *_loop0_213_rule(Parser *p); +static asdl_seq *_loop1_214_rule(Parser *p); +static void *_tmp_215_rule(Parser *p); +static asdl_seq *_loop0_216_rule(Parser *p); +static asdl_seq *_loop1_217_rule(Parser *p); +static void *_tmp_218_rule(Parser *p); static void *_tmp_219_rule(Parser *p); static void *_tmp_220_rule(Parser *p); static void *_tmp_221_rule(Parser *p); @@ -1072,9 +1071,9 @@ static void *_tmp_224_rule(Parser *p); static void *_tmp_225_rule(Parser *p); static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); -static void *_tmp_228_rule(Parser *p); -static asdl_seq *_loop0_230_rule(Parser *p); -static asdl_seq *_gather_229_rule(Parser *p); +static asdl_seq *_loop0_229_rule(Parser *p); +static asdl_seq *_gather_228_rule(Parser *p); +static void *_tmp_230_rule(Parser *p); static void *_tmp_231_rule(Parser *p); static void *_tmp_232_rule(Parser *p); static void *_tmp_233_rule(Parser *p); @@ -1087,8 +1086,8 @@ static void *_tmp_239_rule(Parser *p); static void *_tmp_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); -static void *_tmp_243_rule(Parser *p); -static asdl_seq *_loop0_244_rule(Parser *p); +static asdl_seq *_loop0_243_rule(Parser *p); +static void *_tmp_244_rule(Parser *p); static void *_tmp_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); static void *_tmp_247_rule(Parser *p); @@ -1120,7 +1119,6 @@ static void *_tmp_272_rule(Parser *p); static void *_tmp_273_rule(Parser *p); static void *_tmp_274_rule(Parser *p); static void *_tmp_275_rule(Parser *p); -static void *_tmp_276_rule(Parser *p); // file: statements? $ @@ -20276,7 +20274,7 @@ invalid_legacy_expression_rule(Parser *p) // invalid_expression: // | !(NAME STRING | SOFT_KEYWORD) disjunction expression_without_invalid // | disjunction 'if' disjunction !('else' | ':') -// | 'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field) +// | 'lambda' lambda_params? ':' &FSTRING_MIDDLE static void * invalid_expression_rule(Parser *p) { @@ -20350,12 +20348,12 @@ invalid_expression_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_expression[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); } - { // 'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field) + { // 'lambda' lambda_params? ':' &FSTRING_MIDDLE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field)")); + D(fprintf(stderr, "%*c> invalid_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -20367,10 +20365,10 @@ invalid_expression_rule(Parser *p) && (b = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_157_rule, p) + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, FSTRING_MIDDLE) // token=FSTRING_MIDDLE ) { - D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field)")); + D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "f-string: lambda expressions are not allowed without parentheses" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -20381,7 +20379,7 @@ invalid_expression_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_expression[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'lambda' lambda_params? ':' &(FSTRING_MIDDLE | fstring_replacement_field)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); } _res = NULL; done: @@ -20455,7 +20453,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_158_rule, p) + _PyPegen_lookahead(0, _tmp_157_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20481,7 +20479,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, _tmp_159_rule, p) + _PyPegen_lookahead(0, _tmp_158_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20489,7 +20487,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_160_rule, p) + _PyPegen_lookahead(0, _tmp_159_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20569,7 +20567,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_161_var; + asdl_seq * _loop0_160_var; expr_ty a; expr_ty expression_var; if ( @@ -20577,7 +20575,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_161_var = _loop0_161_rule(p)) // star_named_expressions* + (_loop0_160_var = _loop0_160_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20634,10 +20632,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_162_var; + asdl_seq * _loop0_161_var; expr_ty a; if ( - (_loop0_162_var = _loop0_162_rule(p)) // ((star_targets '='))* + (_loop0_161_var = _loop0_161_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20664,10 +20662,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_162_var; expr_ty a; if ( - (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* + (_loop0_162_var = _loop0_162_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -20693,7 +20691,7 @@ invalid_assignment_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); - void *_tmp_164_var; + void *_tmp_163_var; expr_ty a; AugOperator* augassign_var; if ( @@ -20701,7 +20699,7 @@ invalid_assignment_rule(Parser *p) && (augassign_var = augassign_rule(p)) // augassign && - (_tmp_164_var = _tmp_164_rule(p)) // yield_expr | star_expressions + (_tmp_163_var = _tmp_163_rule(p)) // yield_expr | star_expressions ) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); @@ -20923,11 +20921,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_165_var; + void *_tmp_164_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_165_var = _tmp_165_rule(p)) // '[' | '(' | '{' + (_tmp_164_var = _tmp_164_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -20954,12 +20952,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_166_var; + void *_tmp_165_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' + (_tmp_165_var = _tmp_165_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -20989,12 +20987,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_167_var; + void *_tmp_166_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' + (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21129,13 +21127,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_169_var; - void *_tmp_168_var; + asdl_seq * _loop0_168_var; + void *_tmp_167_var; Token * a; if ( - (_tmp_168_var = _tmp_168_rule(p)) // slash_no_default | slash_with_default + (_tmp_167_var = _tmp_167_rule(p)) // slash_no_default | slash_with_default && - (_loop0_169_var = _loop0_169_rule(p)) // param_maybe_default* + (_loop0_168_var = _loop0_168_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21159,7 +21157,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_170_var; + asdl_seq * _loop0_169_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21167,7 +21165,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* + (_loop0_169_var = _loop0_169_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -21193,18 +21191,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_171_var; - asdl_seq * _loop1_172_var; + asdl_seq * _loop0_170_var; + asdl_seq * _loop1_171_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* + (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_172_var = _loop1_172_rule(p)) // param_no_default+ + (_loop1_171_var = _loop1_171_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21231,22 +21229,22 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_174_var; - asdl_seq * _loop0_176_var; + asdl_seq * _loop0_173_var; + asdl_seq * _loop0_175_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_175_var; + void *_tmp_174_var; Token * a; if ( - (_opt_var = _tmp_173_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_172_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_174_var = _loop0_174_rule(p)) // param_maybe_default* + (_loop0_173_var = _loop0_173_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_175_var = _tmp_175_rule(p)) // ',' | param_no_default + (_tmp_174_var = _tmp_174_rule(p)) // ',' | param_no_default && - (_loop0_176_var = _loop0_176_rule(p)) // param_maybe_default* + (_loop0_175_var = _loop0_175_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21271,10 +21269,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_177_var; + asdl_seq * _loop1_176_var; Token * a; if ( - (_loop1_177_var = _loop1_177_rule(p)) // param_maybe_default+ + (_loop1_176_var = _loop1_176_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21323,7 +21321,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_178_rule, p) + _PyPegen_lookahead(1, _tmp_177_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -21368,12 +21366,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_179_var; + void *_tmp_178_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_179_var = _tmp_179_rule(p)) // ')' | ',' (')' | '**') + (_tmp_178_var = _tmp_178_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -21456,20 +21454,20 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_181_var; - void *_tmp_180_var; - void *_tmp_182_var; + asdl_seq * _loop0_180_var; + void *_tmp_179_var; + void *_tmp_181_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_180_var = _tmp_180_rule(p)) // param_no_default | ',' + (_tmp_179_var = _tmp_179_rule(p)) // param_no_default | ',' && - (_loop0_181_var = _loop0_181_rule(p)) // param_maybe_default* + (_loop0_180_var = _loop0_180_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_182_var = _tmp_182_rule(p)) // param_no_default | ',' + (_tmp_181_var = _tmp_181_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -21584,7 +21582,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_183_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_182_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -21649,13 +21647,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_184_var; + asdl_seq * _loop1_183_var; if ( - (_loop1_184_var = _loop1_184_rule(p)) // param_with_default+ + (_loop1_183_var = _loop1_183_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_184_var; + _res = _loop1_183_var; goto done; } p->mark = _mark; @@ -21720,13 +21718,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_186_var; - void *_tmp_185_var; + asdl_seq * _loop0_185_var; + void *_tmp_184_var; Token * a; if ( - (_tmp_185_var = _tmp_185_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_184_var = _tmp_184_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_maybe_default* + (_loop0_185_var = _loop0_185_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21750,7 +21748,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_187_var; + asdl_seq * _loop0_186_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21758,7 +21756,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* + (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -21784,18 +21782,18 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_189_var; - asdl_seq * _loop0_188_var; + asdl_seq * _gather_188_var; + asdl_seq * _loop0_187_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* + (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_189_var = _gather_189_rule(p)) // ','.lambda_param+ + (_gather_188_var = _gather_188_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21822,22 +21820,22 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_192_var; - asdl_seq * _loop0_194_var; + asdl_seq * _loop0_191_var; + asdl_seq * _loop0_193_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_193_var; + void *_tmp_192_var; Token * a; if ( - (_opt_var = _tmp_191_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_190_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_192_var = _loop0_192_rule(p)) // lambda_param_maybe_default* + (_loop0_191_var = _loop0_191_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_193_var = _tmp_193_rule(p)) // ',' | lambda_param_no_default + (_tmp_192_var = _tmp_192_rule(p)) // ',' | lambda_param_no_default && - (_loop0_194_var = _loop0_194_rule(p)) // lambda_param_maybe_default* + (_loop0_193_var = _loop0_193_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21862,10 +21860,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_195_var; + asdl_seq * _loop1_194_var; Token * a; if ( - (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_maybe_default+ + (_loop1_194_var = _loop1_194_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21936,13 +21934,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_196_var; + asdl_seq * _loop1_195_var; if ( - (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_with_default+ + (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_196_var; + _res = _loop1_195_var; goto done; } p->mark = _mark; @@ -21978,11 +21976,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_197_var; + void *_tmp_196_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_197_var = _tmp_197_rule(p)) // ':' | ',' (':' | '**') + (_tmp_196_var = _tmp_196_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -22035,20 +22033,20 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_199_var; - void *_tmp_198_var; - void *_tmp_200_var; + asdl_seq * _loop0_198_var; + void *_tmp_197_var; + void *_tmp_199_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_198_var = _tmp_198_rule(p)) // lambda_param_no_default | ',' + (_tmp_197_var = _tmp_197_rule(p)) // lambda_param_no_default | ',' && - (_loop0_199_var = _loop0_199_rule(p)) // lambda_param_maybe_default* + (_loop0_198_var = _loop0_198_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_200_var = _tmp_200_rule(p)) // lambda_param_no_default | ',' + (_tmp_199_var = _tmp_199_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -22166,7 +22164,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_201_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_200_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -22272,7 +22270,7 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, _tmp_202_rule, p) + _PyPegen_lookahead(1, _tmp_201_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22445,14 +22443,14 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_203_var; + asdl_seq * _gather_202_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( (a = _PyPegen_expect_token(p, 617)) // token='import' && - (_gather_203_var = _gather_203_rule(p)) // ','.dotted_name+ + (_gather_202_var = _gather_202_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 618)) // token='from' && @@ -22548,7 +22546,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_205_var; + asdl_seq * _gather_204_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22558,7 +22556,7 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 629)) // token='with' && - (_gather_205_var = _gather_205_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_204_var = _gather_204_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22582,7 +22580,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_207_var; + asdl_seq * _gather_206_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22598,7 +22596,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_207_var = _gather_207_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_206_var = _gather_206_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22647,7 +22645,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_209_var; + asdl_seq * _gather_208_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22658,7 +22656,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 629)) // token='with' && - (_gather_209_var = _gather_209_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_208_var = _gather_208_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22686,7 +22684,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_211_var; + asdl_seq * _gather_210_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -22703,7 +22701,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_211_var = _gather_211_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_210_var = _gather_210_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22800,7 +22798,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_213_rule, p) + _PyPegen_lookahead(0, _tmp_212_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -22825,8 +22823,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_214_var; - asdl_seq * _loop1_215_var; + asdl_seq * _loop0_213_var; + asdl_seq * _loop1_214_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -22837,9 +22835,9 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_214_var = _loop0_214_rule(p)) // block* + (_loop0_213_var = _loop0_213_rule(p)) // block* && - (_loop1_215_var = _loop1_215_rule(p)) // except_block+ + (_loop1_214_var = _loop1_214_rule(p)) // except_block+ && (a = _PyPegen_expect_token(p, 651)) // token='except' && @@ -22847,7 +22845,7 @@ invalid_try_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_216_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_215_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22874,8 +22872,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_217_var; - asdl_seq * _loop1_218_var; + asdl_seq * _loop0_216_var; + asdl_seq * _loop1_217_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -22884,13 +22882,13 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_217_var = _loop0_217_rule(p)) // block* + (_loop0_216_var = _loop0_216_rule(p)) // block* && - (_loop1_218_var = _loop1_218_rule(p)) // except_star_block+ + (_loop1_217_var = _loop1_217_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 651)) // token='except' && - (_opt_var = _tmp_219_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_218_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22957,7 +22955,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_219_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22995,7 +22993,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23047,14 +23045,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_222_var; + void *_tmp_221_var; Token * a; if ( (a = _PyPegen_expect_token(p, 651)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_222_var = _tmp_222_rule(p)) // NEWLINE | ':' + (_tmp_221_var = _tmp_221_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23159,7 +23157,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23253,7 +23251,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23617,7 +23615,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_225_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_224_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -24105,7 +24103,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_226_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_2 = _tmp_225_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24164,7 +24162,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_226_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24199,7 +24197,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24249,11 +24247,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_229_var; + asdl_seq * _gather_228_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_229_var = _gather_229_rule(p)) // ','.double_starred_kvpair+ + (_gather_228_var = _gather_228_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24261,7 +24259,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_229_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_228_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24314,7 +24312,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_231_rule, p) + _PyPegen_lookahead(1, _tmp_230_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24424,7 +24422,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_232_rule, p) + _PyPegen_lookahead(1, _tmp_231_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24640,7 +24638,7 @@ invalid_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, _tmp_233_rule, p) + _PyPegen_lookahead(0, _tmp_232_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); @@ -24663,13 +24661,13 @@ invalid_replacement_field_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); Token * _literal; - void *_tmp_234_var; + void *_tmp_233_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_234_var = _tmp_234_rule(p)) // yield_expr | star_expressions + (_tmp_233_var = _tmp_233_rule(p)) // yield_expr | star_expressions && - _PyPegen_lookahead(0, _tmp_235_rule, p) + _PyPegen_lookahead(0, _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); @@ -24693,15 +24691,15 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); Token * _literal; Token * _literal_1; - void *_tmp_236_var; + void *_tmp_235_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_236_var = _tmp_236_rule(p)) // yield_expr | star_expressions + (_tmp_235_var = _tmp_235_rule(p)) // yield_expr | star_expressions && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_237_rule, p) + _PyPegen_lookahead(0, _tmp_236_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); @@ -24726,12 +24724,12 @@ invalid_replacement_field_rule(Parser *p) Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_238_var; + void *_tmp_237_var; void *invalid_conversion_character_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions + (_tmp_237_var = _tmp_237_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && @@ -24739,7 +24737,7 @@ invalid_replacement_field_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_238_var, _opt_var, invalid_conversion_character_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_237_var, _opt_var, invalid_conversion_character_var); goto done; } p->mark = _mark; @@ -24757,17 +24755,17 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_239_var; + void *_tmp_238_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions + (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_239_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_241_rule, p) + _PyPegen_lookahead(0, _tmp_240_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); @@ -24791,24 +24789,24 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_244_var; + asdl_seq * _loop0_243_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_242_var; + void *_tmp_241_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_242_var = _tmp_242_rule(p)) // yield_expr | star_expressions + (_tmp_241_var = _tmp_241_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_242_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_244_var = _loop0_244_rule(p)) // fstring_format_spec* + (_loop0_243_var = _loop0_243_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -24837,15 +24835,15 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_245_var; + void *_tmp_244_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_245_var = _tmp_245_rule(p)) // yield_expr | star_expressions + (_tmp_244_var = _tmp_244_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_246_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_245_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -24892,7 +24890,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_247_rule, p) + _PyPegen_lookahead(1, _tmp_246_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25822,12 +25820,12 @@ _loop1_15_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_248_var; + void *_tmp_247_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // star_targets '=' + (_tmp_247_var = _tmp_247_rule(p)) // star_targets '=' ) { - _res = _tmp_248_var; + _res = _tmp_247_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26391,12 +26389,12 @@ _loop0_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_249_var; + void *_tmp_248_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' + (_tmp_248_var = _tmp_248_rule(p)) // '.' | '...' ) { - _res = _tmp_249_var; + _res = _tmp_248_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26458,12 +26456,12 @@ _loop1_26_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_250_var; + void *_tmp_249_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // '.' | '...' + (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' ) { - _res = _tmp_250_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26856,12 +26854,12 @@ _loop1_33_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_251_var; + void *_tmp_250_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // '@' named_expression NEWLINE + (_tmp_250_var = _tmp_250_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_251_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29986,12 +29984,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_252_var; + void *_tmp_251_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // ',' expression + (_tmp_251_var = _tmp_251_rule(p)) // ',' expression ) { - _res = _tmp_252_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30058,9 +30056,198 @@ _loop1_84_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); + void *_tmp_252_var; + while ( + (_tmp_252_var = _tmp_252_rule(p)) // ',' star_expression + ) + { + _res = _tmp_252_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_84[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop0_86: ',' star_named_expression +static asdl_seq * +_loop0_86_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' star_named_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); + Token * _literal; + expr_ty elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = star_named_expression_rule(p)) // star_named_expression + ) + { + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _gather_85: star_named_expression _loop0_86 +static asdl_seq * +_gather_85_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // star_named_expression _loop0_86 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); + expr_ty elem; + asdl_seq * seq; + if ( + (elem = star_named_expression_rule(p)) // star_named_expression + && + (seq = _loop0_86_rule(p)) // _loop0_86 + ) + { + D(fprintf(stderr, "%*c+ _gather_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_85[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_86")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop1_87: ('or' conjunction) +static asdl_seq * +_loop1_87_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ('or' conjunction) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); void *_tmp_253_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // ',' star_expression + (_tmp_253_var = _tmp_253_rule(p)) // 'or' conjunction ) { _res = _tmp_253_var; @@ -30080,195 +30267,6 @@ _loop1_84_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_84[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_86: ',' star_named_expression -static asdl_seq * -_loop0_86_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' star_named_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = star_named_expression_rule(p)) // star_named_expression - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_85: star_named_expression _loop0_86 -static asdl_seq * -_gather_85_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // star_named_expression _loop0_86 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = star_named_expression_rule(p)) // star_named_expression - && - (seq = _loop0_86_rule(p)) // _loop0_86 - ) - { - D(fprintf(stderr, "%*c+ _gather_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_86")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_85[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_86")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_87: ('or' conjunction) -static asdl_seq * -_loop1_87_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ('or' conjunction) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_254_var; - while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'or' conjunction - ) - { - _res = _tmp_254_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_87[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('or' conjunction)")); } @@ -30319,12 +30317,12 @@ _loop1_88_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_255_var; + void *_tmp_254_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // 'and' inversion + (_tmp_254_var = _tmp_254_rule(p)) // 'and' inversion ) { - _res = _tmp_255_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30511,7 +30509,7 @@ _loop0_92_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_255_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30576,7 +30574,7 @@ _gather_91_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_255_rule(p)) // slice | starred_expression && (seq = _loop0_92_rule(p)) // _loop0_92 ) @@ -32108,12 +32106,12 @@ _loop1_115_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_257_var; + void *_tmp_256_var; while ( - (_tmp_257_var = _tmp_257_rule(p)) // fstring | string + (_tmp_256_var = _tmp_256_rule(p)) // fstring | string ) { - _res = _tmp_257_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32418,12 +32416,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_258_var; + void *_tmp_257_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction + (_tmp_257_var = _tmp_257_rule(p)) // 'if' disjunction ) { - _res = _tmp_258_var; + _res = _tmp_257_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32485,12 +32483,12 @@ _loop0_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_259_var; + void *_tmp_258_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // 'if' disjunction + (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction ) { - _res = _tmp_259_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32616,7 +32614,7 @@ _loop0_124_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_259_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -32682,7 +32680,7 @@ _gather_123_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_259_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_124_rule(p)) // _loop0_124 ) @@ -33243,12 +33241,12 @@ _loop0_134_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_261_var; + void *_tmp_260_var; while ( - (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target + (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target ) { - _res = _tmp_261_var; + _res = _tmp_260_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33427,12 +33425,12 @@ _loop1_137_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_262_var; + void *_tmp_261_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // ',' star_target + (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target ) { - _res = _tmp_262_var; + _res = _tmp_261_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34529,66 +34527,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: FSTRING_MIDDLE | fstring_replacement_field +// _tmp_157: '=' | ':=' static void * _tmp_157_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // FSTRING_MIDDLE - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE")); - Token * fstring_middle_var; - if ( - (fstring_middle_var = _PyPegen_expect_token(p, FSTRING_MIDDLE)) // token='FSTRING_MIDDLE' - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE")); - _res = fstring_middle_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "FSTRING_MIDDLE")); - } - { // fstring_replacement_field - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_replacement_field")); - expr_ty fstring_replacement_field_var; - if ( - (fstring_replacement_field_var = fstring_replacement_field_rule(p)) // fstring_replacement_field - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring_replacement_field")); - _res = fstring_replacement_field_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_replacement_field")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_158: '=' | ':=' -static void * -_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34604,18 +34545,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34623,18 +34564,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34643,9 +34584,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_158: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_159_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34661,18 +34602,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -34680,18 +34621,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -34699,18 +34640,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -34718,18 +34659,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -34737,18 +34678,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -34756,18 +34697,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -34776,9 +34717,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: '=' | ':=' +// _tmp_159: '=' | ':=' static void * -_tmp_160_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34794,18 +34735,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34813,18 +34754,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34833,9 +34774,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _loop0_161: star_named_expressions +// _loop0_160: star_named_expressions static asdl_seq * -_loop0_161_rule(Parser *p) +_loop0_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34860,7 +34801,7 @@ _loop0_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -34883,7 +34824,7 @@ _loop0_161_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34900,9 +34841,9 @@ _loop0_161_rule(Parser *p) return _seq; } -// _loop0_162: (star_targets '=') +// _loop0_161: (star_targets '=') static asdl_seq * -_loop0_162_rule(Parser *p) +_loop0_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34927,13 +34868,13 @@ _loop0_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_263_var; + D(fprintf(stderr, "%*c> _loop0_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_262_var; while ( - (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' + (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' ) { - _res = _tmp_263_var; + _res = _tmp_262_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34950,7 +34891,7 @@ _loop0_162_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34967,9 +34908,9 @@ _loop0_162_rule(Parser *p) return _seq; } -// _loop0_163: (star_targets '=') +// _loop0_162: (star_targets '=') static asdl_seq * -_loop0_163_rule(Parser *p) +_loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34994,13 +34935,13 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_264_var; + D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_263_var; while ( - (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_264_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35017,7 +34958,7 @@ _loop0_163_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35034,9 +34975,9 @@ _loop0_163_rule(Parser *p) return _seq; } -// _tmp_164: yield_expr | star_expressions +// _tmp_163: yield_expr | star_expressions static void * -_tmp_164_rule(Parser *p) +_tmp_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35052,18 +34993,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -35071,18 +35012,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -35091,9 +35032,9 @@ _tmp_164_rule(Parser *p) return _res; } -// _tmp_165: '[' | '(' | '{' +// _tmp_164: '[' | '(' | '{' static void * -_tmp_165_rule(Parser *p) +_tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35109,18 +35050,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -35128,18 +35069,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -35147,18 +35088,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35167,9 +35108,9 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: '[' | '{' +// _tmp_165: '[' | '{' static void * -_tmp_166_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35185,18 +35126,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35204,18 +35145,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35224,9 +35165,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: '[' | '{' +// _tmp_166: '[' | '{' static void * -_tmp_167_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35242,18 +35183,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35261,18 +35202,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35281,9 +35222,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: slash_no_default | slash_with_default +// _tmp_167: slash_no_default | slash_with_default static void * -_tmp_168_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35299,18 +35240,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35318,18 +35259,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35338,9 +35279,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _loop0_169: param_maybe_default +// _loop0_168: param_maybe_default static asdl_seq * -_loop0_169_rule(Parser *p) +_loop0_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35365,7 +35306,7 @@ _loop0_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35388,7 +35329,7 @@ _loop0_169_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35405,9 +35346,9 @@ _loop0_169_rule(Parser *p) return _seq; } -// _loop0_170: param_no_default +// _loop0_169: param_no_default static asdl_seq * -_loop0_170_rule(Parser *p) +_loop0_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35432,7 +35373,7 @@ _loop0_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35455,7 +35396,7 @@ _loop0_170_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35472,9 +35413,9 @@ _loop0_170_rule(Parser *p) return _seq; } -// _loop0_171: param_no_default +// _loop0_170: param_no_default static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35499,7 +35440,7 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35522,7 +35463,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35539,9 +35480,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _loop1_172: param_no_default +// _loop1_171: param_no_default static asdl_seq * -_loop1_172_rule(Parser *p) +_loop1_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35566,7 +35507,7 @@ _loop1_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35589,7 +35530,7 @@ _loop1_172_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -35611,9 +35552,9 @@ _loop1_172_rule(Parser *p) return _seq; } -// _tmp_173: slash_no_default | slash_with_default +// _tmp_172: slash_no_default | slash_with_default static void * -_tmp_173_rule(Parser *p) +_tmp_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35629,18 +35570,18 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35648,18 +35589,18 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35668,9 +35609,9 @@ _tmp_173_rule(Parser *p) return _res; } -// _loop0_174: param_maybe_default +// _loop0_173: param_maybe_default static asdl_seq * -_loop0_174_rule(Parser *p) +_loop0_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35695,7 +35636,7 @@ _loop0_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35718,7 +35659,7 @@ _loop0_174_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35735,9 +35676,9 @@ _loop0_174_rule(Parser *p) return _seq; } -// _tmp_175: ',' | param_no_default +// _tmp_174: ',' | param_no_default static void * -_tmp_175_rule(Parser *p) +_tmp_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35753,18 +35694,18 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -35772,18 +35713,18 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -35792,9 +35733,9 @@ _tmp_175_rule(Parser *p) return _res; } -// _loop0_176: param_maybe_default +// _loop0_175: param_maybe_default static asdl_seq * -_loop0_176_rule(Parser *p) +_loop0_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35819,7 +35760,7 @@ _loop0_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35842,7 +35783,7 @@ _loop0_176_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35859,9 +35800,9 @@ _loop0_176_rule(Parser *p) return _seq; } -// _loop1_177: param_maybe_default +// _loop1_176: param_maybe_default static asdl_seq * -_loop1_177_rule(Parser *p) +_loop1_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35886,7 +35827,7 @@ _loop1_177_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35909,7 +35850,7 @@ _loop1_177_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_177[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -35931,9 +35872,9 @@ _loop1_177_rule(Parser *p) return _seq; } -// _tmp_178: ')' | ',' +// _tmp_177: ')' | ',' static void * -_tmp_178_rule(Parser *p) +_tmp_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35949,18 +35890,18 @@ _tmp_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -35968,18 +35909,18 @@ _tmp_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -35988,9 +35929,9 @@ _tmp_178_rule(Parser *p) return _res; } -// _tmp_179: ')' | ',' (')' | '**') +// _tmp_178: ')' | ',' (')' | '**') static void * -_tmp_179_rule(Parser *p) +_tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36006,18 +35947,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -36025,21 +35966,21 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_265_var; + void *_tmp_264_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_265_var = _tmp_265_rule(p)) // ')' | '**' + (_tmp_264_var = _tmp_264_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -36048,9 +35989,9 @@ _tmp_179_rule(Parser *p) return _res; } -// _tmp_180: param_no_default | ',' +// _tmp_179: param_no_default | ',' static void * -_tmp_180_rule(Parser *p) +_tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36066,18 +36007,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36085,18 +36026,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36105,9 +36046,9 @@ _tmp_180_rule(Parser *p) return _res; } -// _loop0_181: param_maybe_default +// _loop0_180: param_maybe_default static asdl_seq * -_loop0_181_rule(Parser *p) +_loop0_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36132,7 +36073,7 @@ _loop0_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36155,7 +36096,7 @@ _loop0_181_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36172,9 +36113,9 @@ _loop0_181_rule(Parser *p) return _seq; } -// _tmp_182: param_no_default | ',' +// _tmp_181: param_no_default | ',' static void * -_tmp_182_rule(Parser *p) +_tmp_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36190,18 +36131,18 @@ _tmp_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36209,18 +36150,18 @@ _tmp_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36229,9 +36170,9 @@ _tmp_182_rule(Parser *p) return _res; } -// _tmp_183: '*' | '**' | '/' +// _tmp_182: '*' | '**' | '/' static void * -_tmp_183_rule(Parser *p) +_tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36247,18 +36188,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -36266,18 +36207,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -36285,18 +36226,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -36305,9 +36246,9 @@ _tmp_183_rule(Parser *p) return _res; } -// _loop1_184: param_with_default +// _loop1_183: param_with_default static asdl_seq * -_loop1_184_rule(Parser *p) +_loop1_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36332,7 +36273,7 @@ _loop1_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -36355,7 +36296,7 @@ _loop1_184_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -36377,9 +36318,9 @@ _loop1_184_rule(Parser *p) return _seq; } -// _tmp_185: lambda_slash_no_default | lambda_slash_with_default +// _tmp_184: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_185_rule(Parser *p) +_tmp_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36395,18 +36336,18 @@ _tmp_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36414,18 +36355,18 @@ _tmp_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36434,9 +36375,9 @@ _tmp_185_rule(Parser *p) return _res; } -// _loop0_186: lambda_param_maybe_default +// _loop0_185: lambda_param_maybe_default static asdl_seq * -_loop0_186_rule(Parser *p) +_loop0_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36461,7 +36402,7 @@ _loop0_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36484,7 +36425,7 @@ _loop0_186_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36501,9 +36442,9 @@ _loop0_186_rule(Parser *p) return _seq; } -// _loop0_187: lambda_param_no_default +// _loop0_186: lambda_param_no_default static asdl_seq * -_loop0_187_rule(Parser *p) +_loop0_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36528,7 +36469,7 @@ _loop0_187_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36551,7 +36492,7 @@ _loop0_187_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36568,9 +36509,9 @@ _loop0_187_rule(Parser *p) return _seq; } -// _loop0_188: lambda_param_no_default +// _loop0_187: lambda_param_no_default static asdl_seq * -_loop0_188_rule(Parser *p) +_loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36595,7 +36536,7 @@ _loop0_188_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36618,7 +36559,7 @@ _loop0_188_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36635,9 +36576,9 @@ _loop0_188_rule(Parser *p) return _seq; } -// _loop0_190: ',' lambda_param +// _loop0_189: ',' lambda_param static asdl_seq * -_loop0_190_rule(Parser *p) +_loop0_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36662,7 +36603,7 @@ _loop0_190_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -36694,7 +36635,7 @@ _loop0_190_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_190[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_189[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36711,9 +36652,9 @@ _loop0_190_rule(Parser *p) return _seq; } -// _gather_189: lambda_param _loop0_190 +// _gather_188: lambda_param _loop0_189 static asdl_seq * -_gather_189_rule(Parser *p) +_gather_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36724,27 +36665,27 @@ _gather_189_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_190 + { // lambda_param _loop0_189 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c> _gather_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_189")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_190_rule(p)) // _loop0_190 + (seq = _loop0_189_rule(p)) // _loop0_189 ) { - D(fprintf(stderr, "%*c+ _gather_189[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c+ _gather_188[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_189")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_189[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_190")); + D(fprintf(stderr, "%*c%s _gather_188[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_189")); } _res = NULL; done: @@ -36752,9 +36693,9 @@ _gather_189_rule(Parser *p) return _res; } -// _tmp_191: lambda_slash_no_default | lambda_slash_with_default +// _tmp_190: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_191_rule(Parser *p) +_tmp_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36770,18 +36711,18 @@ _tmp_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_190[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36789,18 +36730,18 @@ _tmp_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_190[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36809,9 +36750,9 @@ _tmp_191_rule(Parser *p) return _res; } -// _loop0_192: lambda_param_maybe_default +// _loop0_191: lambda_param_maybe_default static asdl_seq * -_loop0_192_rule(Parser *p) +_loop0_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36836,7 +36777,7 @@ _loop0_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36859,7 +36800,7 @@ _loop0_192_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_191[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36876,9 +36817,9 @@ _loop0_192_rule(Parser *p) return _seq; } -// _tmp_193: ',' | lambda_param_no_default +// _tmp_192: ',' | lambda_param_no_default static void * -_tmp_193_rule(Parser *p) +_tmp_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36894,18 +36835,18 @@ _tmp_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -36913,18 +36854,18 @@ _tmp_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -36933,9 +36874,9 @@ _tmp_193_rule(Parser *p) return _res; } -// _loop0_194: lambda_param_maybe_default +// _loop0_193: lambda_param_maybe_default static asdl_seq * -_loop0_194_rule(Parser *p) +_loop0_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36960,7 +36901,7 @@ _loop0_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36983,7 +36924,7 @@ _loop0_194_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_193[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37000,9 +36941,9 @@ _loop0_194_rule(Parser *p) return _seq; } -// _loop1_195: lambda_param_maybe_default +// _loop1_194: lambda_param_maybe_default static asdl_seq * -_loop1_195_rule(Parser *p) +_loop1_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37027,7 +36968,7 @@ _loop1_195_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37050,7 +36991,7 @@ _loop1_195_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -37072,9 +37013,9 @@ _loop1_195_rule(Parser *p) return _seq; } -// _loop1_196: lambda_param_with_default +// _loop1_195: lambda_param_with_default static asdl_seq * -_loop1_196_rule(Parser *p) +_loop1_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37099,7 +37040,7 @@ _loop1_196_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -37122,7 +37063,7 @@ _loop1_196_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -37144,9 +37085,9 @@ _loop1_196_rule(Parser *p) return _seq; } -// _tmp_197: ':' | ',' (':' | '**') +// _tmp_196: ':' | ',' (':' | '**') static void * -_tmp_197_rule(Parser *p) +_tmp_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37162,18 +37103,18 @@ _tmp_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_196[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -37181,21 +37122,21 @@ _tmp_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_266_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_266_var = _tmp_266_rule(p)) // ':' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); + D(fprintf(stderr, "%*c+ _tmp_196[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -37204,9 +37145,9 @@ _tmp_197_rule(Parser *p) return _res; } -// _tmp_198: lambda_param_no_default | ',' +// _tmp_197: lambda_param_no_default | ',' static void * -_tmp_198_rule(Parser *p) +_tmp_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37222,18 +37163,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37241,18 +37182,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37261,9 +37202,9 @@ _tmp_198_rule(Parser *p) return _res; } -// _loop0_199: lambda_param_maybe_default +// _loop0_198: lambda_param_maybe_default static asdl_seq * -_loop0_199_rule(Parser *p) +_loop0_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37288,7 +37229,7 @@ _loop0_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37311,7 +37252,7 @@ _loop0_199_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37328,9 +37269,9 @@ _loop0_199_rule(Parser *p) return _seq; } -// _tmp_200: lambda_param_no_default | ',' +// _tmp_199: lambda_param_no_default | ',' static void * -_tmp_200_rule(Parser *p) +_tmp_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37346,18 +37287,18 @@ _tmp_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37365,18 +37306,18 @@ _tmp_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37385,9 +37326,9 @@ _tmp_200_rule(Parser *p) return _res; } -// _tmp_201: '*' | '**' | '/' +// _tmp_200: '*' | '**' | '/' static void * -_tmp_201_rule(Parser *p) +_tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37403,18 +37344,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -37422,18 +37363,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -37441,18 +37382,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -37461,9 +37402,9 @@ _tmp_201_rule(Parser *p) return _res; } -// _tmp_202: ',' | ')' | ':' +// _tmp_201: ',' | ')' | ':' static void * -_tmp_202_rule(Parser *p) +_tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37479,18 +37420,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37498,18 +37439,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -37517,18 +37458,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -37537,9 +37478,9 @@ _tmp_202_rule(Parser *p) return _res; } -// _loop0_204: ',' dotted_name +// _loop0_203: ',' dotted_name static asdl_seq * -_loop0_204_rule(Parser *p) +_loop0_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37564,7 +37505,7 @@ _loop0_204_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -37596,7 +37537,7 @@ _loop0_204_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_204[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37613,9 +37554,9 @@ _loop0_204_rule(Parser *p) return _seq; } -// _gather_203: dotted_name _loop0_204 +// _gather_202: dotted_name _loop0_203 static asdl_seq * -_gather_203_rule(Parser *p) +_gather_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37626,27 +37567,27 @@ _gather_203_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_204 + { // dotted_name _loop0_203 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c> _gather_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_203")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_204_rule(p)) // _loop0_204 + (seq = _loop0_203_rule(p)) // _loop0_203 ) { - D(fprintf(stderr, "%*c+ _gather_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c+ _gather_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_203")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_203[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c%s _gather_202[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_203")); } _res = NULL; done: @@ -37654,9 +37595,9 @@ _gather_203_rule(Parser *p) return _res; } -// _loop0_206: ',' (expression ['as' star_target]) +// _loop0_205: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_206_rule(Parser *p) +_loop0_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37681,13 +37622,13 @@ _loop0_206_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_267_rule(p)) // expression ['as' star_target] + (elem = _tmp_266_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37713,7 +37654,7 @@ _loop0_206_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_206[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37730,9 +37671,9 @@ _loop0_206_rule(Parser *p) return _seq; } -// _gather_205: (expression ['as' star_target]) _loop0_206 +// _gather_204: (expression ['as' star_target]) _loop0_205 static asdl_seq * -_gather_205_rule(Parser *p) +_gather_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37743,27 +37684,27 @@ _gather_205_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_206 + { // (expression ['as' star_target]) _loop0_205 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c> _gather_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_205")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_267_rule(p)) // expression ['as' star_target] + (elem = _tmp_266_rule(p)) // expression ['as' star_target] && - (seq = _loop0_206_rule(p)) // _loop0_206 + (seq = _loop0_205_rule(p)) // _loop0_205 ) { - D(fprintf(stderr, "%*c+ _gather_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c+ _gather_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_205")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_205[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c%s _gather_204[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_205")); } _res = NULL; done: @@ -37771,9 +37712,9 @@ _gather_205_rule(Parser *p) return _res; } -// _loop0_208: ',' (expressions ['as' star_target]) +// _loop0_207: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_208_rule(Parser *p) +_loop0_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37798,13 +37739,13 @@ _loop0_208_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expressions ['as' star_target] + (elem = _tmp_267_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -37830,7 +37771,7 @@ _loop0_208_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_208[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37847,9 +37788,9 @@ _loop0_208_rule(Parser *p) return _seq; } -// _gather_207: (expressions ['as' star_target]) _loop0_208 +// _gather_206: (expressions ['as' star_target]) _loop0_207 static asdl_seq * -_gather_207_rule(Parser *p) +_gather_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37860,27 +37801,27 @@ _gather_207_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_208 + { // (expressions ['as' star_target]) _loop0_207 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_207")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expressions ['as' star_target] + (elem = _tmp_267_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_208_rule(p)) // _loop0_208 + (seq = _loop0_207_rule(p)) // _loop0_207 ) { - D(fprintf(stderr, "%*c+ _gather_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_207")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_207[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_207")); } _res = NULL; done: @@ -37888,9 +37829,9 @@ _gather_207_rule(Parser *p) return _res; } -// _loop0_210: ',' (expression ['as' star_target]) +// _loop0_209: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_210_rule(Parser *p) +_loop0_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37915,13 +37856,13 @@ _loop0_210_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_269_rule(p)) // expression ['as' star_target] + (elem = _tmp_268_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37947,7 +37888,7 @@ _loop0_210_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_210[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37964,9 +37905,9 @@ _loop0_210_rule(Parser *p) return _seq; } -// _gather_209: (expression ['as' star_target]) _loop0_210 +// _gather_208: (expression ['as' star_target]) _loop0_209 static asdl_seq * -_gather_209_rule(Parser *p) +_gather_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37977,27 +37918,27 @@ _gather_209_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_210 + { // (expression ['as' star_target]) _loop0_209 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_269_rule(p)) // expression ['as' star_target] + (elem = _tmp_268_rule(p)) // expression ['as' star_target] && - (seq = _loop0_210_rule(p)) // _loop0_210 + (seq = _loop0_209_rule(p)) // _loop0_209 ) { - D(fprintf(stderr, "%*c+ _gather_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_209[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); } _res = NULL; done: @@ -38005,9 +37946,9 @@ _gather_209_rule(Parser *p) return _res; } -// _loop0_212: ',' (expressions ['as' star_target]) +// _loop0_211: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_212_rule(Parser *p) +_loop0_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38032,13 +37973,13 @@ _loop0_212_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_270_rule(p)) // expressions ['as' star_target] + (elem = _tmp_269_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38064,7 +38005,7 @@ _loop0_212_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_212[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38081,9 +38022,9 @@ _loop0_212_rule(Parser *p) return _seq; } -// _gather_211: (expressions ['as' star_target]) _loop0_212 +// _gather_210: (expressions ['as' star_target]) _loop0_211 static asdl_seq * -_gather_211_rule(Parser *p) +_gather_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38094,27 +38035,27 @@ _gather_211_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_212 + { // (expressions ['as' star_target]) _loop0_211 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_270_rule(p)) // expressions ['as' star_target] + (elem = _tmp_269_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_212_rule(p)) // _loop0_212 + (seq = _loop0_211_rule(p)) // _loop0_211 ) { - D(fprintf(stderr, "%*c+ _gather_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); } _res = NULL; done: @@ -38122,9 +38063,9 @@ _gather_211_rule(Parser *p) return _res; } -// _tmp_213: 'except' | 'finally' +// _tmp_212: 'except' | 'finally' static void * -_tmp_213_rule(Parser *p) +_tmp_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38140,18 +38081,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 651)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_212[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -38159,18 +38100,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 647)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_212[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -38179,9 +38120,9 @@ _tmp_213_rule(Parser *p) return _res; } -// _loop0_214: block +// _loop0_213: block static asdl_seq * -_loop0_214_rule(Parser *p) +_loop0_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38206,7 +38147,7 @@ _loop0_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38229,7 +38170,7 @@ _loop0_214_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38246,9 +38187,9 @@ _loop0_214_rule(Parser *p) return _seq; } -// _loop1_215: except_block +// _loop1_214: except_block static asdl_seq * -_loop1_215_rule(Parser *p) +_loop1_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38273,7 +38214,7 @@ _loop1_215_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -38296,7 +38237,7 @@ _loop1_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_215[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -38318,9 +38259,9 @@ _loop1_215_rule(Parser *p) return _seq; } -// _tmp_216: 'as' NAME +// _tmp_215: 'as' NAME static void * -_tmp_216_rule(Parser *p) +_tmp_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38336,7 +38277,7 @@ _tmp_216_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38345,12 +38286,12 @@ _tmp_216_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_215[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_215[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38359,9 +38300,9 @@ _tmp_216_rule(Parser *p) return _res; } -// _loop0_217: block +// _loop0_216: block static asdl_seq * -_loop0_217_rule(Parser *p) +_loop0_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38386,7 +38327,7 @@ _loop0_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38409,7 +38350,7 @@ _loop0_217_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38426,9 +38367,9 @@ _loop0_217_rule(Parser *p) return _seq; } -// _loop1_218: except_star_block +// _loop1_217: except_star_block static asdl_seq * -_loop1_218_rule(Parser *p) +_loop1_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38453,7 +38394,7 @@ _loop1_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -38476,7 +38417,7 @@ _loop1_218_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_217[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -38498,9 +38439,9 @@ _loop1_218_rule(Parser *p) return _seq; } -// _tmp_219: expression ['as' NAME] +// _tmp_218: expression ['as' NAME] static void * -_tmp_219_rule(Parser *p) +_tmp_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38516,22 +38457,22 @@ _tmp_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_270_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_218[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_218[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -38540,9 +38481,9 @@ _tmp_219_rule(Parser *p) return _res; } -// _tmp_220: 'as' NAME +// _tmp_219: 'as' NAME static void * -_tmp_220_rule(Parser *p) +_tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38558,7 +38499,7 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38567,12 +38508,12 @@ _tmp_220_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38581,9 +38522,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: 'as' NAME +// _tmp_220: 'as' NAME static void * -_tmp_221_rule(Parser *p) +_tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38599,7 +38540,7 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38608,12 +38549,12 @@ _tmp_221_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38622,9 +38563,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: NEWLINE | ':' +// _tmp_221: NEWLINE | ':' static void * -_tmp_222_rule(Parser *p) +_tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38640,18 +38581,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -38659,18 +38600,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -38679,9 +38620,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: 'as' NAME +// _tmp_222: 'as' NAME static void * -_tmp_223_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38697,7 +38638,7 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38706,12 +38647,12 @@ _tmp_223_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38720,9 +38661,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: 'as' NAME +// _tmp_223: 'as' NAME static void * -_tmp_224_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38738,7 +38679,7 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38747,12 +38688,12 @@ _tmp_224_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38761,9 +38702,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: positional_patterns ',' +// _tmp_224: positional_patterns ',' static void * -_tmp_225_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38779,7 +38720,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -38788,12 +38729,12 @@ _tmp_225_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -38802,9 +38743,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: '->' expression +// _tmp_225: '->' expression static void * -_tmp_226_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38820,7 +38761,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -38829,12 +38770,12 @@ _tmp_226_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -38843,9 +38784,9 @@ _tmp_226_rule(Parser *p) return _res; } -// _tmp_227: '(' arguments? ')' +// _tmp_226: '(' arguments? ')' static void * -_tmp_227_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38861,7 +38802,7 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -38874,12 +38815,12 @@ _tmp_227_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -38888,9 +38829,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_227: '(' arguments? ')' static void * -_tmp_228_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38906,7 +38847,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -38919,12 +38860,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -38933,9 +38874,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _loop0_230: ',' double_starred_kvpair +// _loop0_229: ',' double_starred_kvpair static asdl_seq * -_loop0_230_rule(Parser *p) +_loop0_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38960,7 +38901,7 @@ _loop0_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -38992,7 +38933,7 @@ _loop0_230_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39009,9 +38950,9 @@ _loop0_230_rule(Parser *p) return _seq; } -// _gather_229: double_starred_kvpair _loop0_230 +// _gather_228: double_starred_kvpair _loop0_229 static asdl_seq * -_gather_229_rule(Parser *p) +_gather_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39022,27 +38963,27 @@ _gather_229_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_230 + { // double_starred_kvpair _loop0_229 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c> _gather_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_229")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_230_rule(p)) // _loop0_230 + (seq = _loop0_229_rule(p)) // _loop0_229 ) { - D(fprintf(stderr, "%*c+ _gather_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c+ _gather_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_229")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_229[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c%s _gather_228[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_229")); } _res = NULL; done: @@ -39050,9 +38991,9 @@ _gather_229_rule(Parser *p) return _res; } -// _tmp_231: '}' | ',' +// _tmp_230: '}' | ',' static void * -_tmp_231_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39068,18 +39009,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39087,18 +39028,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39107,9 +39048,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_231: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39125,18 +39066,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39144,18 +39085,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39164,9 +39105,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: yield_expr | star_expressions +// _tmp_232: yield_expr | star_expressions static void * -_tmp_233_rule(Parser *p) +_tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39182,18 +39123,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39201,18 +39142,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39221,9 +39162,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: yield_expr | star_expressions +// _tmp_233: yield_expr | star_expressions static void * -_tmp_234_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39239,18 +39180,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39258,18 +39199,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39278,9 +39219,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: '=' | '!' | ':' | '}' +// _tmp_234: '=' | '!' | ':' | '}' static void * -_tmp_235_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39296,18 +39237,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39315,18 +39256,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39334,18 +39275,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39353,18 +39294,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39373,9 +39314,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: yield_expr | star_expressions +// _tmp_235: yield_expr | star_expressions static void * -_tmp_236_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39391,18 +39332,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39410,18 +39351,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39430,9 +39371,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: '!' | ':' | '}' +// _tmp_236: '!' | ':' | '}' static void * -_tmp_237_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39448,18 +39389,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39467,18 +39408,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39486,18 +39427,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39506,9 +39447,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: yield_expr | star_expressions +// _tmp_237: yield_expr | star_expressions static void * -_tmp_238_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39524,18 +39465,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39543,18 +39484,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39563,9 +39504,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: yield_expr | star_expressions +// _tmp_238: yield_expr | star_expressions static void * -_tmp_239_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39581,18 +39522,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39600,18 +39541,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39620,9 +39561,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: '!' NAME +// _tmp_239: '!' NAME static void * -_tmp_240_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39638,7 +39579,7 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39647,12 +39588,12 @@ _tmp_240_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39661,9 +39602,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: ':' | '}' +// _tmp_240: ':' | '}' static void * -_tmp_241_rule(Parser *p) +_tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39679,18 +39620,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39698,18 +39639,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39718,9 +39659,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: yield_expr | star_expressions +// _tmp_241: yield_expr | star_expressions static void * -_tmp_242_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39736,18 +39677,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39755,18 +39696,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39775,9 +39716,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: '!' NAME +// _tmp_242: '!' NAME static void * -_tmp_243_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39793,7 +39734,7 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39802,12 +39743,12 @@ _tmp_243_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39816,9 +39757,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _loop0_244: fstring_format_spec +// _loop0_243: fstring_format_spec static asdl_seq * -_loop0_244_rule(Parser *p) +_loop0_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39843,7 +39784,7 @@ _loop0_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -39866,7 +39807,7 @@ _loop0_244_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39883,9 +39824,9 @@ _loop0_244_rule(Parser *p) return _seq; } -// _tmp_245: yield_expr | star_expressions +// _tmp_244: yield_expr | star_expressions static void * -_tmp_245_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39901,18 +39842,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39920,18 +39861,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39940,9 +39881,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '!' NAME +// _tmp_245: '!' NAME static void * -_tmp_246_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39958,7 +39899,7 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39967,12 +39908,12 @@ _tmp_246_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39981,9 +39922,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: ':' | '}' +// _tmp_246: ':' | '}' static void * -_tmp_247_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39999,18 +39940,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40018,18 +39959,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40038,9 +39979,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: star_targets '=' +// _tmp_247: star_targets '=' static void * -_tmp_248_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40056,7 +39997,7 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40065,7 +40006,7 @@ _tmp_248_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40075,7 +40016,7 @@ _tmp_248_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40084,9 +40025,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '.' | '...' +// _tmp_248: '.' | '...' static void * -_tmp_249_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40102,18 +40043,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40121,18 +40062,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40141,9 +40082,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: '.' | '...' +// _tmp_249: '.' | '...' static void * -_tmp_250_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40159,18 +40100,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40178,18 +40119,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40198,9 +40139,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: '@' named_expression NEWLINE +// _tmp_250: '@' named_expression NEWLINE static void * -_tmp_251_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40216,7 +40157,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40228,7 +40169,7 @@ _tmp_251_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40238,7 +40179,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40247,9 +40188,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: ',' expression +// _tmp_251: ',' expression static void * -_tmp_252_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40265,7 +40206,7 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40274,7 +40215,7 @@ _tmp_252_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40284,7 +40225,7 @@ _tmp_252_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40293,9 +40234,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: ',' star_expression +// _tmp_252: ',' star_expression static void * -_tmp_253_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40311,7 +40252,7 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40320,7 +40261,7 @@ _tmp_253_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40330,7 +40271,7 @@ _tmp_253_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40339,9 +40280,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'or' conjunction +// _tmp_253: 'or' conjunction static void * -_tmp_254_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40357,7 +40298,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40366,7 +40307,7 @@ _tmp_254_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40376,7 +40317,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40385,9 +40326,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: 'and' inversion +// _tmp_254: 'and' inversion static void * -_tmp_255_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40403,7 +40344,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40412,7 +40353,7 @@ _tmp_255_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40422,7 +40363,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40431,9 +40372,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: slice | starred_expression +// _tmp_255: slice | starred_expression static void * -_tmp_256_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40449,18 +40390,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40468,18 +40409,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40488,9 +40429,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _tmp_257: fstring | string +// _tmp_256: fstring | string static void * -_tmp_257_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40506,18 +40447,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40525,18 +40466,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40545,9 +40486,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: 'if' disjunction +// _tmp_257: 'if' disjunction static void * -_tmp_258_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40563,7 +40504,7 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40572,7 +40513,7 @@ _tmp_258_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40582,7 +40523,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40591,9 +40532,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: 'if' disjunction +// _tmp_258: 'if' disjunction static void * -_tmp_259_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40609,7 +40550,7 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40618,7 +40559,7 @@ _tmp_259_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40628,7 +40569,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40637,9 +40578,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_259: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_260_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40655,18 +40596,18 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -40674,20 +40615,20 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_272_var; + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_271_var; if ( - (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' + (_tmp_271_var = _tmp_271_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_272_var; + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_271_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -40696,9 +40637,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: ',' star_target +// _tmp_260: ',' star_target static void * -_tmp_261_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40714,7 +40655,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40723,7 +40664,7 @@ _tmp_261_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40733,7 +40674,7 @@ _tmp_261_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40742,9 +40683,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: ',' star_target +// _tmp_261: ',' star_target static void * -_tmp_262_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40760,7 +40701,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40769,7 +40710,7 @@ _tmp_262_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40779,7 +40720,7 @@ _tmp_262_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40788,9 +40729,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: star_targets '=' +// _tmp_262: star_targets '=' static void * -_tmp_263_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40806,7 +40747,7 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40815,12 +40756,12 @@ _tmp_263_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40829,9 +40770,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_264_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40847,7 +40788,7 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40856,12 +40797,12 @@ _tmp_264_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40870,9 +40811,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: ')' | '**' +// _tmp_264: ')' | '**' static void * -_tmp_265_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40888,18 +40829,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -40907,18 +40848,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -40927,9 +40868,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ':' | '**' +// _tmp_265: ':' | '**' static void * -_tmp_266_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40945,18 +40886,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -40964,18 +40905,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -40984,9 +40925,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: expression ['as' star_target] +// _tmp_266: expression ['as' star_target] static void * -_tmp_267_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41002,22 +40943,22 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_272_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41026,9 +40967,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expressions ['as' star_target] +// _tmp_267: expressions ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41044,22 +40985,22 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41068,9 +41009,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: expression ['as' star_target] +// _tmp_268: expression ['as' star_target] static void * -_tmp_269_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41086,22 +41027,22 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41110,9 +41051,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: expressions ['as' star_target] +// _tmp_269: expressions ['as' star_target] static void * -_tmp_270_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41128,22 +41069,22 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41152,9 +41093,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: 'as' NAME +// _tmp_270: 'as' NAME static void * -_tmp_271_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41170,7 +41111,7 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -41179,12 +41120,12 @@ _tmp_271_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41193,9 +41134,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: assignment_expression | expression !':=' +// _tmp_271: assignment_expression | expression !':=' static void * -_tmp_272_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41211,18 +41152,18 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41230,7 +41171,7 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41238,12 +41179,12 @@ _tmp_272_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41252,9 +41193,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: 'as' star_target +// _tmp_272: 'as' star_target static void * -_tmp_273_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41270,7 +41211,7 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41279,12 +41220,12 @@ _tmp_273_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41293,9 +41234,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _tmp_274: 'as' star_target +// _tmp_273: 'as' star_target static void * -_tmp_274_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41311,7 +41252,7 @@ _tmp_274_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41320,12 +41261,12 @@ _tmp_274_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41334,9 +41275,9 @@ _tmp_274_rule(Parser *p) return _res; } -// _tmp_275: 'as' star_target +// _tmp_274: 'as' star_target static void * -_tmp_275_rule(Parser *p) +_tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41352,7 +41293,7 @@ _tmp_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41361,12 +41302,12 @@ _tmp_275_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41375,9 +41316,9 @@ _tmp_275_rule(Parser *p) return _res; } -// _tmp_276: 'as' star_target +// _tmp_275: 'as' star_target static void * -_tmp_276_rule(Parser *p) +_tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41393,7 +41334,7 @@ _tmp_276_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41402,12 +41343,12 @@ _tmp_276_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; From 15d659f929bb2a79fa7211947ef4f2c43818fd31 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 22:06:34 +0200 Subject: [PATCH 571/598] gh-91960: FreeBSD Cirrus CI runs configure separately (#109127) Run configure and make in separated steps to have more readable logs. --- .cirrus.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 823b1f921d66d1..ca41c2e9092fa6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,13 +10,16 @@ freebsd_task: sysctl_script: - sysctl net.inet.tcp.blackhole=0 - sysctl net.inet.udp.blackhole=0 - build_script: + configure_script: - mkdir build - cd build - ../configure --with-pydebug + build_script: + - cd build - make -j$(sysctl -n hw.ncpu) pythoninfo_script: - - cd build && make pythoninfo + - cd build + - make pythoninfo test_script: - cd build # dtrace fails to build on FreeBSD - see gh-73263 From 697c9dcf8fc746636c6187e4f110e0e6e865b710 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 8 Sep 2023 22:05:40 +0100 Subject: [PATCH 572/598] gh-108455: peg_generator: enable mypy's `--warn-unreachable` setting and `redundant-expr` error code (#109160) --- Tools/peg_generator/mypy.ini | 14 +++++++++++--- Tools/peg_generator/pegen/ast_dump.py | 2 -- Tools/peg_generator/pegen/grammar.py | 6 ++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini index 8820d77b36f646..f38f21bed004b2 100644 --- a/Tools/peg_generator/mypy.ini +++ b/Tools/peg_generator/mypy.ini @@ -8,8 +8,16 @@ python_version = 3.10 # Be strict... strict = True -enable_error_code = truthy-bool,ignore-without-code +warn_unreachable = True +enable_error_code = truthy-bool,ignore-without-code,redundant-expr -# except for a few settings that can't yet be enabled: +# This causes *many* false positives on the peg_generator +# due to pegen.grammar.GrammarVisitor returning Any from visit() and generic_visit(). +# It would be possible to workaround the false positives using asserts, +# but it would be pretty tedious, and probably isn't worth it. warn_return_any = False -warn_unreachable = False + +# Not all of the strictest settings can be enabled +# on generated Python code yet: +[mypy-pegen.grammar_parser.*] +disable_error_code = redundant-expr diff --git a/Tools/peg_generator/pegen/ast_dump.py b/Tools/peg_generator/pegen/ast_dump.py index 2c57d0932fda37..07f8799c114f5d 100644 --- a/Tools/peg_generator/pegen/ast_dump.py +++ b/Tools/peg_generator/pegen/ast_dump.py @@ -66,6 +66,4 @@ def _format(node: Any, level: int = 0) -> Tuple[str, bool]: if all(cls.__name__ != "AST" for cls in node.__class__.__mro__): raise TypeError("expected AST, got %r" % node.__class__.__name__) - if indent is not None and not isinstance(indent, str): - indent = " " * indent return _format(node)[0] diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index fcf868eb1753e5..065894e7fe66ab 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -112,8 +112,7 @@ def __str__(self) -> str: return self.value def __iter__(self) -> Iterable[str]: - if False: - yield + yield from () class NameLeaf(Leaf): @@ -335,8 +334,7 @@ def __str__(self) -> str: return f"~" def __iter__(self) -> Iterator[Tuple[str, str]]: - if False: - yield + yield from () def __eq__(self, other: object) -> bool: if not isinstance(other, Cut): From 1f7e42131d2800f0fbb89bfd91fafa8a073e066d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Sep 2023 23:14:33 +0200 Subject: [PATCH 573/598] gh-109054: configure checks if libatomic is needed (#109101) Fix building the _testcapi extension on Linux AArch64 which requires linking to libatomic when is used: the _Py_atomic_or_uint64() function requires libatomic __atomic_fetch_or_8() on this platform. The configure script now checks if linking to libatomic is needed and generates a new LIBATOMIC variable used to build the _testcapi extension. Building the _testcapi extension now uses the LIBATOMIC variable in its LDFLAGS, since Modules/_testcapi/pyatomic.c uses . Co-authored-by: Erlend E. Aasland --- ...-09-07-19-58-05.gh-issue-109054.5r3S3l.rst | 6 ++ configure | 85 ++++++++++++++++++- configure.ac | 62 +++++++++++++- 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst diff --git a/Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst b/Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst new file mode 100644 index 00000000000000..d86a110e0de68c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-09-07-19-58-05.gh-issue-109054.5r3S3l.rst @@ -0,0 +1,6 @@ +Fix building the ``_testcapi`` extension on Linux AArch64 which requires +linking to libatomic when ```` is used: the +``_Py_atomic_or_uint64()`` function requires libatomic +``__atomic_fetch_or_8()`` on this platform. The configure script now checks +if linking to libatomic is needed and generates a new LIBATOMIC variable +used to build the _testcapi extension. Patch by Victor Stinner. diff --git a/configure b/configure index d73b4b271ac719..c78c45d11260f6 100755 --- a/configure +++ b/configure @@ -27752,6 +27752,88 @@ printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h fi +# gh-109054: Check if -latomic is needed to get atomic functions. +# On Linux aarch64, GCC may require programs and libraries to be linked +# explicitly to libatomic. Call _Py_atomic_or_uint64() which may require +# libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the +# compiler flags. +# +# Avoid #include or #include . The header +# requires header which is only written below by AC_OUTPUT below. +# If the check is done after AC_OUTPUT, modifying LIBATOMIC has no effect +# anymore. cannot be included alone, it's designed to be included +# by : it expects other includes and macros to be defined. +save_CPPFLAGS=$CPPFLAGS +CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether libatomic is needed by " >&5 +printf %s "checking whether libatomic is needed by ... " >&6; } +if test ${ac_cv_libatomic_needed+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test "$cross_compiling" = yes +then : + ac_cv_libatomic_needed=yes +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +// pyatomic.h needs uint64_t and Py_ssize_t types +#include // int64_t, intptr_t +#ifdef HAVE_SYS_TYPES_H +# include // ssize_t +#endif +// Code adapted from Include/pyport.h +#if HAVE_SSIZE_T +typedef ssize_t Py_ssize_t; +#elif SIZEOF_VOID_P == SIZEOF_SIZE_T +typedef intptr_t Py_ssize_t; +#else +# error "unable to define Py_ssize_t" +#endif + +#include "cpython/pyatomic.h" + +int main() +{ + uint64_t byte; + _Py_atomic_store_uint64(&byte, 2); + if (_Py_atomic_or_uint64(&byte, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&byte) != 10) { + return 1; // error + } + return 0; // all good +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + ac_cv_libatomic_needed=no + +else $as_nop + ac_cv_libatomic_needed=yes +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 +printf "%s\n" "$ac_cv_libatomic_needed" >&6; } + +if test "x$ac_cv_libatomic_needed" = xyes +then : + LIBATOMIC=${LIBATOMIC-"-latomic"} +fi +CPPFLAGS=$save_CPPFLAGS + + +# stdlib # stdlib not available @@ -29900,7 +29982,7 @@ fi then : - + as_fn_append MODULE_BLOCK "MODULE__TESTCAPI_LDFLAGS=$LIBATOMIC$as_nl" fi if test "$py_cv_module__testcapi" = yes; then @@ -30344,6 +30426,7 @@ ac_config_files="$ac_config_files Modules/Setup.bootstrap Modules/Setup.stdlib" ac_config_files="$ac_config_files Modules/ld_so_aix" +# Generate files like pyconfig.h cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure # tests run on this system so they can be shared between configure diff --git a/configure.ac b/configure.ac index 612c072af329d4..f833755a466012 100644 --- a/configure.ac +++ b/configure.ac @@ -6962,6 +6962,62 @@ then [Define if you want to disable the GIL]) fi +# gh-109054: Check if -latomic is needed to get atomic functions. +# On Linux aarch64, GCC may require programs and libraries to be linked +# explicitly to libatomic. Call _Py_atomic_or_uint64() which may require +# libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the +# compiler flags. +# +# Avoid #include or #include . The header +# requires header which is only written below by AC_OUTPUT below. +# If the check is done after AC_OUTPUT, modifying LIBATOMIC has no effect +# anymore. cannot be included alone, it's designed to be included +# by : it expects other includes and macros to be defined. +_SAVE_VAR([CPPFLAGS]) +CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" + +AC_CACHE_CHECK([whether libatomic is needed by ], + [ac_cv_libatomic_needed], +[AC_RUN_IFELSE([AC_LANG_SOURCE([[ +// pyatomic.h needs uint64_t and Py_ssize_t types +#include // int64_t, intptr_t +#ifdef HAVE_SYS_TYPES_H +# include // ssize_t +#endif +// Code adapted from Include/pyport.h +#if HAVE_SSIZE_T +typedef ssize_t Py_ssize_t; +#elif SIZEOF_VOID_P == SIZEOF_SIZE_T +typedef intptr_t Py_ssize_t; +#else +# error "unable to define Py_ssize_t" +#endif + +#include "cpython/pyatomic.h" + +int main() +{ + uint64_t byte; + _Py_atomic_store_uint64(&byte, 2); + if (_Py_atomic_or_uint64(&byte, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&byte) != 10) { + return 1; // error + } + return 0; // all good +} +]])],[ + ac_cv_libatomic_needed=no +],[ac_cv_libatomic_needed=yes],[ac_cv_libatomic_needed=yes]) +]) + +AS_VAR_IF([ac_cv_libatomic_needed], [yes], + [LIBATOMIC=${LIBATOMIC-"-latomic"}]) +_RESTORE_VAR([CPPFLAGS]) + + +# stdlib AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ AS_VAR_SET([py_cv_module_]mod, [n/a])]) @@ -7229,7 +7285,10 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules -PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testcapi], + [test "$TEST_MODULES" = yes], [] + dnl Modules/_testcapi/pyatomic.c uses header + [], [], [$LIBATOMIC]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) @@ -7262,6 +7321,7 @@ AC_CONFIG_FILES(m4_normalize([ Modules/Setup.stdlib ])) AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix]) +# Generate files like pyconfig.h AC_OUTPUT AC_MSG_NOTICE([creating Modules/Setup.local]) From bcb2ab5ef8c646565b09c860fb14e415d7b374bd Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 9 Sep 2023 06:38:38 +0800 Subject: [PATCH 574/598] gh-108996: add tests for msvcrt (#109004) Co-authored-by: Terry Jan Reedy Co-authored-by: Steve Dower --- Lib/test/test_msvcrt.py | 111 ++++++++++++++++++ ...-09-06-22-06-22.gh-issue-108996.IBhR3U.rst | 1 + 2 files changed, 112 insertions(+) create mode 100644 Lib/test/test_msvcrt.py create mode 100644 Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py new file mode 100644 index 00000000000000..adf4e26b98ac55 --- /dev/null +++ b/Lib/test/test_msvcrt.py @@ -0,0 +1,111 @@ +import os +import sys +import unittest + +from test.support import os_helper +from test.support.os_helper import TESTFN, TESTFN_ASCII + +if sys.platform != "win32": + raise unittest.SkipTest("windows related tests") + +import _winapi +import msvcrt; + +from _testconsole import write_input + + +class TestFileOperations(unittest.TestCase): + def test_locking(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + self.assertRaises(OSError, msvcrt.locking, f.fileno(), msvcrt.LK_NBLCK, 1) + + def test_unlockfile(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + + def test_setmode(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.setmode(f.fileno(), os.O_BINARY) + msvcrt.setmode(f.fileno(), os.O_TEXT) + + def test_open_osfhandle(self): + h = _winapi.CreateFile(TESTFN_ASCII, _winapi.GENERIC_WRITE, 0, 0, 1, 128, 0) + self.addCleanup(os_helper.unlink, TESTFN_ASCII) + + try: + fd = msvcrt.open_osfhandle(h, os.O_RDONLY) + h = None + os.close(fd) + finally: + if h: + _winapi.CloseHandle(h) + + def test_get_osfhandle(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.get_osfhandle(f.fileno()) + + +c = '\u5b57' # unicode CJK char (meaning 'character') for 'wide-char' tests +c_encoded = b'\x57\x5b' # utf-16-le (which windows internally used) encoded char for this CJK char + + +class TestConsoleIO(unittest.TestCase): + def test_kbhit(self): + self.assertEqual(msvcrt.kbhit(), 0) + + def test_getch(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getch(), b'c') + + def test_getwch(self): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input(stdin.buffer.raw, c_encoded) + self.assertEqual(msvcrt.getwch(), c) + finally: + sys.stdin = old_stdin + + def test_getche(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getche(), b'c') + + def test_getwche(self): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input(stdin.buffer.raw, c_encoded) + self.assertEqual(msvcrt.getwche(), c) + finally: + sys.stdin = old_stdin + + def test_putch(self): + msvcrt.putch(b'c') + + def test_putwch(self): + msvcrt.putwch(c) + + +class TestOther(unittest.TestCase): + def test_heap_min(self): + try: + msvcrt.heapmin() + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst b/Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst new file mode 100644 index 00000000000000..887f8b74bcfa30 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-06-22-06-22.gh-issue-108996.IBhR3U.rst @@ -0,0 +1 @@ +Add tests for ``msvcrt``. From 5b7303e2653a0723a3e4c767d03dd02681206ca8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 00:41:26 +0200 Subject: [PATCH 575/598] gh-109162: Refactor Regrtest.main() (#109163) * main() now calls _parse_args() and pass 'ns' to Regrtest constructor. Remove kwargs argument from Regrtest.main(). * _parse_args() checks ns.huntrleaks. * set_temp_dir() is now responsible to call expanduser(). * Regrtest.main() sets self.tests earlier. * Add TestTuple and TestList types. * Rename MatchTests to FilterTuple and rename MatchTestsDict to FilterTestDict. * TestResult.get_rerun_match_tests() return type is now FilterTuple: return a tuple instead of a list. RunTests.tests type becomes TestTuple. --- Lib/test/libregrtest/cmdline.py | 9 ++++ Lib/test/libregrtest/main.py | 87 +++++++++++++----------------- Lib/test/libregrtest/runtest.py | 26 +++++---- Lib/test/libregrtest/runtest_mp.py | 4 +- 4 files changed, 64 insertions(+), 62 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d1a590d8c1a5b3..bbac980a0bf5d8 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -448,4 +448,13 @@ def _parse_args(args, **kwargs): # --forever implies --failfast ns.failfast = True + if ns.huntrleaks: + warmup, repetitions, _ = ns.huntrleaks + if warmup < 1 or repetitions < 1: + msg = ("Invalid values for the --huntrleaks/-R parameters. The " + "number of warmups and repetitions must be at least 1 " + "each (1:1).") + print(msg, file=sys.stderr, flush=True) + sys.exit(2) + return ns diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ab03647ca5802f..f973f03ab26512 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -9,10 +9,10 @@ import tempfile import time import unittest -from test.libregrtest.cmdline import _parse_args +from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, runtest, abs_module_name, - PROGRESS_MIN_TIME, State, MatchTestsDict, RunTests) + PROGRESS_MIN_TIME, State, FilterDict, RunTests, TestResult, TestList) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, @@ -58,9 +58,9 @@ class Regrtest: directly to set the values that would normally be set by flags on the command line. """ - def __init__(self): + def __init__(self, ns: Namespace): # Namespace of command line options - self.ns = None + self.ns: Namespace = ns # tests self.tests = [] @@ -68,14 +68,14 @@ def __init__(self): self.all_runtests: list[RunTests] = [] # test results - self.good: list[str] = [] - self.bad: list[str] = [] - self.rerun_bad: list[str] = [] - self.skipped: list[str] = [] - self.resource_denied: list[str] = [] - self.environment_changed: list[str] = [] - self.run_no_tests: list[str] = [] - self.rerun: list[str] = [] + self.good: TestList = [] + self.bad: TestList = [] + self.rerun_bad: TestList = [] + self.skipped: TestList = [] + self.resource_denied: TestList = [] + self.environment_changed: TestList = [] + self.run_no_tests: TestList = [] + self.rerun: TestList = [] self.need_rerun: list[TestResult] = [] self.first_state: str | None = None @@ -184,29 +184,7 @@ def display_progress(self, test_index, text): line = f"{line}/{fails}" self.log(f"[{line}] {text}") - def parse_args(self, kwargs): - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.xmlpath: - support.junit_xml_list = self.testsuite_xml = [] - - strip_py_suffix(ns.args) - - if ns.huntrleaks: - warmup, repetitions, _ = ns.huntrleaks - if warmup < 1 or repetitions < 1: - msg = ("Invalid values for the --huntrleaks/-R parameters. The " - "number of warmups and repetitions must be at least 1 " - "each (1:1).") - print(msg, file=sys.stderr, flush=True) - sys.exit(2) - - if ns.tempdir: - ns.tempdir = os.path.expanduser(ns.tempdir) - - self.ns = ns - - def find_tests(self, tests): + def find_tests(self): ns = self.ns single = ns.single fromfile = ns.fromfile @@ -216,8 +194,6 @@ def find_tests(self, tests): starting_test = ns.start randomize = ns.randomize - self.tests = tests - if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: @@ -321,7 +297,7 @@ def list_cases(self): print(count(len(skipped), "test"), "skipped:", file=stderr) printlist(skipped, file=stderr) - def get_rerun_match(self, rerun_list) -> MatchTestsDict: + def get_rerun_match(self, rerun_list) -> FilterDict: rerun_match_tests = {} for result in rerun_list: match_tests = result.get_rerun_match_tests() @@ -352,7 +328,7 @@ def _rerun_failed_tests(self, need_rerun): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = RunTests(tests, match_tests=match_tests, rerun=True) + runtests = RunTests(tuple(tests), match_tests=match_tests, rerun=True) self.all_runtests.append(runtests) self._run_tests_mp(runtests) @@ -624,7 +600,7 @@ def run_tests(self): tests = self.selected self.set_tests(tests) - runtests = RunTests(tests, forever=self.ns.forever) + runtests = RunTests(tuple(tests), forever=self.ns.forever) self.all_runtests.append(runtests) if self.ns.use_mp: self._run_tests_mp(runtests) @@ -737,8 +713,12 @@ def fix_umask(self): os.umask(old_mask) def set_temp_dir(self): - if self.ns.tempdir: - self.tmp_dir = self.ns.tempdir + ns = self.ns + if ns.tempdir: + ns.tempdir = os.path.expanduser(ns.tempdir) + + if ns.tempdir: + self.tmp_dir = ns.tempdir if not self.tmp_dir: # When tests are run from the Python build directory, it is best practice @@ -795,14 +775,20 @@ def cleanup(self): print("Remove file: %s" % name) os_helper.unlink(name) - def main(self, tests=None, **kwargs): - self.parse_args(kwargs) + def main(self, tests: TestList | None = None): + ns = self.ns + self.tests = tests + + if ns.xmlpath: + support.junit_xml_list = self.testsuite_xml = [] + + strip_py_suffix(ns.args) self.set_temp_dir() self.fix_umask() - if self.ns.cleanup: + if ns.cleanup: self.cleanup() sys.exit(0) @@ -817,9 +803,9 @@ def main(self, tests=None, **kwargs): # When using multiprocessing, worker processes will use test_cwd # as their parent temporary directory. So when the main process # exit, it removes also subdirectories of worker processes. - self.ns.tempdir = test_cwd + ns.tempdir = test_cwd - self._main(tests, kwargs) + self._main() except SystemExit as exc: # bpo-38203: Python can hang at exit in Py_Finalize(), especially # on threading._shutdown() call: put a timeout @@ -862,7 +848,7 @@ def action_run_tests(self): self.display_summary() self.finalize() - def _main(self, tests, kwargs): + def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import run_tests_worker run_tests_worker(self.ns.worker_args) @@ -872,7 +858,7 @@ def _main(self, tests, kwargs): input("Press any key to continue...") setup_tests(self.ns) - self.find_tests(tests) + self.find_tests() exitcode = 0 if self.ns.list_tests: @@ -888,4 +874,5 @@ def _main(self, tests, kwargs): def main(tests=None, **kwargs): """Run the Python suite.""" - Regrtest().main(tests=tests, **kwargs) + ns = _parse_args(sys.argv[1:], **kwargs) + Regrtest(ns).main(tests=tests) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 16ae04191da768..7e4b2e6a36b452 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -19,8 +19,13 @@ from test.libregrtest.utils import clear_caches, format_duration, print_warning -MatchTests = list[str] -MatchTestsDict = dict[str, MatchTests] +TestTuple = list[str] +TestList = list[str] + +# --match and --ignore options: list of patterns +# ('*' joker character can be used) +FilterTuple = tuple[str, ...] +FilterDict = dict[str, FilterTuple] # Avoid enum.Enum to reduce the number of imports when tests are run @@ -174,7 +179,7 @@ def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: return True return False - def get_rerun_match_tests(self): + def get_rerun_match_tests(self) -> FilterTuple | None: match_tests = [] errors = self.errors or [] @@ -195,29 +200,30 @@ def get_rerun_match_tests(self): return None match_tests.append(match_name) - return match_tests + if not match_tests: + return None + return tuple(match_tests) @dataclasses.dataclass(slots=True, frozen=True) class RunTests: - tests: list[str] - match_tests: MatchTestsDict | None = None + tests: TestTuple + match_tests: FilterDict | None = None rerun: bool = False forever: bool = False - def get_match_tests(self, test_name) -> MatchTests | None: + def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests is not None: return self.match_tests.get(test_name, None) else: return None def iter_tests(self): - tests = tuple(self.tests) if self.forever: while True: - yield from tests + yield from self.tests else: - yield from tests + yield from self.tests # Minimum duration of a test to display its duration or to mention that diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 60089554cab5dd..2ecdfca0e77010 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -20,7 +20,7 @@ from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( runtest, TestResult, State, PROGRESS_MIN_TIME, - MatchTests, RunTests) + FilterTuple, RunTests) from test.libregrtest.setup import setup_tests from test.libregrtest.utils import format_duration, print_warning @@ -49,7 +49,7 @@ class WorkerJob: test_name: str namespace: Namespace rerun: bool = False - match_tests: MatchTests | None = None + match_tests: FilterTuple | None = None class _EncodeWorkerJob(json.JSONEncoder): From ac8409b38b5a11d88b2cfe4e38a712e8967ec843 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 01:48:54 +0200 Subject: [PATCH 576/598] gh-109162: Regrtest copies 'ns' attributes (#109168) * Regrtest.__init__() now copies 'ns' namespace attributes to Regrtest attributes. Regrtest match_tests and ignore_tests attributes have type FilterTuple (tuple), instead of a list. * Add RunTests.copy(). Regrtest._rerun_failed_tests() now uses RunTests.copy(). * Replace Regrtest.all_tests (list) with Regrtest.first_runtests (RunTests). * Make random_seed maximum 10x larger (9 digits, instead of 8). --- Lib/test/libregrtest/main.py | 114 +++++++++++++++++++------------- Lib/test/libregrtest/runtest.py | 5 ++ Lib/test/test_regrtest.py | 2 +- 3 files changed, 73 insertions(+), 48 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f973f03ab26512..bf71ec3b1b08be 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,7 +12,8 @@ from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, runtest, abs_module_name, - PROGRESS_MIN_TIME, State, FilterDict, RunTests, TestResult, TestList) + PROGRESS_MIN_TIME, State, RunTests, TestResult, + FilterTuple, FilterDict, TestList) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, @@ -62,10 +63,35 @@ def __init__(self, ns: Namespace): # Namespace of command line options self.ns: Namespace = ns + # Actions + self.want_header = ns.header + self.want_list_tests = ns.list_tests + self.want_list_cases = ns.list_cases + self.want_wait = ns.wait + self.want_cleanup = ns.cleanup + + # Select tests + if ns.match_tests: + self.match_tests: FilterTuple = tuple(ns.match_tests) + else: + self.match_tests = None + if ns.ignore_tests: + self.ignore_tests: FilterTuple = tuple(ns.ignore_tests) + else: + self.ignore_tests = None + self.exclude = ns.exclude + self.fromfile = ns.fromfile + self.starting_test = ns.start + + # Options to run tests + self.forever = ns.forever + self.randomize = ns.randomize + self.random_seed = ns.random_seed + # tests self.tests = [] self.selected = [] - self.all_runtests: list[RunTests] = [] + self.first_runtests: RunTests | None = None # test results self.good: TestList = [] @@ -187,12 +213,8 @@ def display_progress(self, test_index, text): def find_tests(self): ns = self.ns single = ns.single - fromfile = ns.fromfile pgo = ns.pgo - exclude = ns.exclude test_dir = ns.testdir - starting_test = ns.start - randomize = ns.randomize if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') @@ -203,12 +225,12 @@ def find_tests(self): except OSError: pass - if fromfile: + if self.fromfile: self.tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') - with open(os.path.join(os_helper.SAVEDCWD, fromfile)) as fp: + with open(os.path.join(os_helper.SAVEDCWD, self.fromfile)) as fp: for line in fp: line = line.split('#', 1)[0] line = line.strip() @@ -223,14 +245,14 @@ def find_tests(self): setup_pgo_tests(ns) exclude_tests = set() - if exclude: + if self.exclude: for arg in ns.args: exclude_tests.add(arg) ns.args = [] alltests = findtests(testdir=test_dir, exclude=exclude_tests) - if not fromfile: + if not self.fromfile: self.selected = self.tests or ns.args if self.selected: self.selected = split_test_packages(self.selected) @@ -248,17 +270,17 @@ def find_tests(self): pass # Remove all the selected tests that precede start if it's set. - if starting_test: + if self.starting_test: try: - del self.selected[:self.selected.index(starting_test)] + del self.selected[:self.selected.index(self.starting_test)] except ValueError: - print(f"Cannot find starting test: {starting_test}") + print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) - if randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) + if self.randomize: + if self.random_seed is None: + self.random_seed = random.randrange(100_000_000) + random.seed(self.random_seed) random.shuffle(self.selected) def list_tests(self): @@ -279,7 +301,7 @@ def list_cases(self): ns = self.ns test_dir = ns.testdir support.verbose = False - support.set_match_tests(ns.match_tests, ns.ignore_tests) + support.set_match_tests(self.match_tests, self.ignore_tests) skipped = [] for test_name in self.selected: @@ -306,20 +328,18 @@ def get_rerun_match(self, rerun_list) -> FilterDict: rerun_match_tests[result.test_name] = match_tests return rerun_match_tests - def _rerun_failed_tests(self, need_rerun): + def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests ns = self.ns ns.verbose = True ns.failfast = False ns.verbose3 = False - ns.forever = False if ns.use_mp is None: ns.use_mp = 1 # Get tests to re-run tests = [result.test_name for result in need_rerun] match_tests = self.get_rerun_match(need_rerun) - self.set_tests(tests) # Clear previously failed tests self.rerun_bad.extend(self.bad) @@ -328,11 +348,14 @@ def _rerun_failed_tests(self, need_rerun): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = RunTests(tuple(tests), match_tests=match_tests, rerun=True) - self.all_runtests.append(runtests) + runtests = runtests.copy(tests=tuple(tests), + match_tests=match_tests, + rerun=True, + forever=False) + self.set_tests(runtests) self._run_tests_mp(runtests) - def rerun_failed_tests(self, need_rerun): + def rerun_failed_tests(self, need_rerun, runtests: RunTests): if self.ns.python: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -344,7 +367,7 @@ def rerun_failed_tests(self, need_rerun): self.first_state = self.get_tests_state() print() - self._rerun_failed_tests(need_rerun) + self._rerun_failed_tests(need_rerun, runtests) if self.bad: print(count(len(self.bad), 'test'), "failed again:") @@ -572,9 +595,9 @@ def _run_tests_mp(self, runtests: RunTests) -> None: self.win_load_tracker.close() self.win_load_tracker = None - def set_tests(self, tests): - self.tests = tests - if self.ns.forever: + def set_tests(self, runtests: RunTests): + self.tests = runtests.tests + if runtests.forever: self.test_count_text = '' self.test_count_width = 3 else: @@ -583,7 +606,7 @@ def set_tests(self, tests): def run_tests(self): # For a partial run, we do not need to clutter the output. - if (self.ns.header + if (self.want_header or not(self.ns.pgo or self.ns.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() @@ -595,17 +618,18 @@ def run_tests(self): "3 warmup repetitions can give false positives!") print(msg, file=sys.stdout, flush=True) - if self.ns.randomize: - print("Using random seed", self.ns.random_seed) + if self.randomize: + print("Using random seed", self.random_seed) tests = self.selected - self.set_tests(tests) - runtests = RunTests(tuple(tests), forever=self.ns.forever) - self.all_runtests.append(runtests) + runtests = RunTests(tuple(tests), forever=self.forever) + self.first_runtests = runtests + self.set_tests(runtests) if self.ns.use_mp: self._run_tests_mp(runtests) else: self.run_tests_sequentially(runtests) + return runtests def finalize(self): if self.next_single_filename: @@ -627,11 +651,7 @@ def finalize(self): def display_summary(self): duration = time.perf_counter() - self.start_time - first_runtests = self.all_runtests[0] - # the second runtests (re-run failed tests) disables forever, - # use the first runtests - forever = first_runtests.forever - filtered = bool(self.ns.match_tests) or bool(self.ns.ignore_tests) + filtered = bool(self.match_tests) or bool(self.ignore_tests) # Total duration print() @@ -655,8 +675,8 @@ def display_summary(self): self.environment_changed, self.run_no_tests] run = sum(map(len, all_tests)) text = f'run={run}' - if not forever: - ntest = len(first_runtests.tests) + if not self.first_runtests.forever: + ntest = len(self.first_runtests.tests) text = f"{text}/{ntest}" if filtered: text = f"{text} (filtered)" @@ -788,7 +808,7 @@ def main(self, tests: TestList | None = None): self.fix_umask() - if ns.cleanup: + if self.want_cleanup: self.cleanup() sys.exit(0) @@ -838,12 +858,12 @@ def get_exitcode(self): return exitcode def action_run_tests(self): - self.run_tests() + runtests = self.run_tests() self.display_result() need_rerun = self.need_rerun if self.ns.rerun and need_rerun: - self.rerun_failed_tests(need_rerun) + self.rerun_failed_tests(need_rerun, runtests) self.display_summary() self.finalize() @@ -854,16 +874,16 @@ def _main(self): run_tests_worker(self.ns.worker_args) return - if self.ns.wait: + if self.want_wait: input("Press any key to continue...") setup_tests(self.ns) self.find_tests() exitcode = 0 - if self.ns.list_tests: + if self.want_list_tests: self.list_tests() - elif self.ns.list_cases: + elif self.want_list_cases: self.list_cases() else: self.action_run_tests() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 7e4b2e6a36b452..5fcbb777bb829b 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -212,6 +212,11 @@ class RunTests: rerun: bool = False forever: bool = False + def copy(self, **override): + state = dataclasses.asdict(self) + state.update(override) + return RunTests(**state) + def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests is not None: return self.match_tests.get(test_name, None) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index aff5404408f8d0..ece8c1f5efdff6 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -589,7 +589,7 @@ def list_regex(line_format, tests): def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 10000000, randseed) + self.assertTrue(0 <= randseed <= 100_000_000, randseed) return randseed def run_command(self, args, input=None, exitcode=0, **kw): From 2fafc3d5c6d720c9b9a4803dde60607fa44b89ce Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 01:56:53 +0200 Subject: [PATCH 577/598] gh-108996: Skip broken test_msvcrt for now (#109169) --- Lib/test/test_msvcrt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index adf4e26b98ac55..3a63de351e095d 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -2,6 +2,8 @@ import sys import unittest +raise unittest.SkipTest("FIXME! broken test see: https://github.com/python/cpython/pull/109004") + from test.support import os_helper from test.support.os_helper import TESTFN, TESTFN_ASCII From 489ca0acf00bb87ae63ab9199787843fdec5a0c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 02:30:28 +0200 Subject: [PATCH 578/598] gh-109162: Refactor Regrtest.action_run_tests() (#109170) Refator Regrtest class: * Rename finalize() finalize_tests(). * Pass tracer to run_test() and finalize_tests(). Remove Regrtest.tracer. * run_test() does less things: move code to its caller. --- Lib/test/libregrtest/main.py | 98 ++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index bf71ec3b1b08be..13500fae50c8e2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -111,9 +111,6 @@ def __init__(self, ns: Namespace): # used by --slow self.test_times = [] - # used by --coverage, trace.Trace instance - self.tracer = None - # used to display the progress bar "[ 3/100]" self.start_time = time.perf_counter() self.test_count_text = '' @@ -443,28 +440,18 @@ def display_result(self): print(count(len(self.run_no_tests), "test"), "run no tests:") printlist(self.run_no_tests) - def run_test(self, test_index, test_name, previous_test, save_modules): - text = test_name - if previous_test: - text = '%s -- %s' % (text, previous_test) - self.display_progress(test_index, text) - - if self.tracer: + def run_test(self, test_name: str, runtests: RunTests, tracer): + if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name); ' - 'self.accumulate_result(result)') + cmd = ('result = runtest(self.ns, test_name)') ns = dict(locals()) - self.tracer.runctx(cmd, globals=globals(), locals=ns) + tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: result = runtest(self.ns, test_name) - self.accumulate_result(result) - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) + self.accumulate_result(result) return result @@ -477,7 +464,9 @@ def run_tests_sequentially(self, runtests): if coverage: import trace - self.tracer = trace.Trace(trace=False, count=True) + tracer = trace.Trace(trace=False, count=True) + else: + tracer = None save_modules = sys.modules.keys() @@ -491,8 +480,17 @@ def run_tests_sequentially(self, runtests): for test_index, test_name in enumerate(tests_iter, 1): start_time = time.perf_counter() - result = self.run_test(test_index, test_name, - previous_test, save_modules) + text = test_name + if previous_test: + text = '%s -- %s' % (text, previous_test) + self.display_progress(test_index, text) + + result = self.run_test(test_name, runtests, tracer) + + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) if result.must_stop(fail_fast, fail_env_changed): break @@ -508,6 +506,8 @@ def run_tests_sequentially(self, runtests): if previous_test: print(previous_test) + return tracer + def display_header(self): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) @@ -604,34 +604,17 @@ def set_tests(self, runtests: RunTests): self.test_count_text = '/{}'.format(len(self.tests)) self.test_count_width = len(self.test_count_text) - 1 - def run_tests(self): - # For a partial run, we do not need to clutter the output. - if (self.want_header - or not(self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - self.display_header() - - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and less than " - "3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - if self.randomize: - print("Using random seed", self.random_seed) - - tests = self.selected - runtests = RunTests(tuple(tests), forever=self.forever) + def run_tests(self, runtests: RunTests): self.first_runtests = runtests self.set_tests(runtests) if self.ns.use_mp: self._run_tests_mp(runtests) + tracer = None else: - self.run_tests_sequentially(runtests) - return runtests + tracer = self.run_tests_sequentially(runtests) + return tracer - def finalize(self): + def finalize_tests(self, tracer): if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -639,10 +622,10 @@ def finalize(self): else: os.unlink(self.next_single_filename) - if self.tracer: - r = self.tracer.results() - r.write_results(show_missing=True, summary=True, - coverdir=self.ns.coverdir) + if tracer is not None: + results = tracer.results() + results.write_results(show_missing=True, summary=True, + coverdir=self.ns.coverdir) if self.ns.runleaks: os.system("leaks %d" % os.getpid()) @@ -858,7 +841,24 @@ def get_exitcode(self): return exitcode def action_run_tests(self): - runtests = self.run_tests() + if self.ns.huntrleaks: + warmup, repetitions, _ = self.ns.huntrleaks + if warmup < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and less than " + "3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.ns.pgo or self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + self.display_header() + + if self.randomize: + print("Using random seed", self.random_seed) + + runtests = RunTests(tuple(self.selected), forever=self.forever) + tracer = self.run_tests(runtests) self.display_result() need_rerun = self.need_rerun @@ -866,7 +866,7 @@ def action_run_tests(self): self.rerun_failed_tests(need_rerun, runtests) self.display_summary() - self.finalize() + self.finalize_tests(tracer) def _main(self): if self.is_worker(): From a56c92875699c2ba92ed49e72f6abbf363a5c537 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 03:03:39 +0200 Subject: [PATCH 579/598] gh-109162: Refactor libregrtest WorkerJob (#109171) * Rename --worker-args command line option to --worker-json. * Rename _parse_worker_args() to _parse_worker_json(). * WorkerJob: * Add runtests attribute * Remove test_name and rerun attribute * Rename run_test_in_subprocess() to create_worker_process(). * Rename run_tests_worker() to worker_process(). * create_worker_process() uses json.dump(): write directly JSON to stdout. * Convert MultiprocessResult to a frozen dataclass. * Rename RunTests.match_tests to RunTests.match_tests_dict. --- Lib/test/libregrtest/cmdline.py | 3 +- Lib/test/libregrtest/main.py | 10 ++--- Lib/test/libregrtest/runtest.py | 6 +-- Lib/test/libregrtest/runtest_mp.py | 62 ++++++++++++++++-------------- Lib/test/test_regrtest.py | 8 ++-- 5 files changed, 48 insertions(+), 41 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index bbac980a0bf5d8..c08b3be9285d9f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -170,6 +170,7 @@ def __init__(self, **kwargs) -> None: self.ignore_tests = None self.pgo = False self.pgo_extended = False + self.worker_json = None super().__init__(**kwargs) @@ -205,7 +206,7 @@ def _create_parser(): group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' 'to be attached') - group.add_argument('--worker-args', metavar='ARGS') + group.add_argument('--worker-json', metavar='ARGS') group.add_argument('-S', '--start', metavar='START', help='the name of the test at which to start.' + more_details) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 13500fae50c8e2..cd053c59a98548 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -336,7 +336,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Get tests to re-run tests = [result.test_name for result in need_rerun] - match_tests = self.get_rerun_match(need_rerun) + match_tests_dict = self.get_rerun_match(need_rerun) # Clear previously failed tests self.rerun_bad.extend(self.bad) @@ -346,7 +346,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy(tests=tuple(tests), - match_tests=match_tests, + match_tests_dict=match_tests_dict, rerun=True, forever=False) self.set_tests(runtests) @@ -742,7 +742,7 @@ def set_temp_dir(self): self.tmp_dir = os.path.abspath(self.tmp_dir) def is_worker(self): - return (self.ns.worker_args is not None) + return (self.ns.worker_json is not None) def create_temp_dir(self): os.makedirs(self.tmp_dir, exist_ok=True) @@ -870,8 +870,8 @@ def action_run_tests(self): def _main(self): if self.is_worker(): - from test.libregrtest.runtest_mp import run_tests_worker - run_tests_worker(self.ns.worker_args) + from test.libregrtest.runtest_mp import worker_process + worker_process(self.ns.worker_json) return if self.want_wait: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 5fcbb777bb829b..ea5d0564b14aea 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -208,7 +208,7 @@ def get_rerun_match_tests(self) -> FilterTuple | None: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple - match_tests: FilterDict | None = None + match_tests_dict: FilterDict | None = None rerun: bool = False forever: bool = False @@ -218,8 +218,8 @@ def copy(self, **override): return RunTests(**state) def get_match_tests(self, test_name) -> FilterTuple | None: - if self.match_tests is not None: - return self.match_tests.get(test_name, None) + if self.match_tests_dict is not None: + return self.match_tests_dict.get(test_name, None) else: return None diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 2ecdfca0e77010..45db24f255fcf7 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -46,9 +46,8 @@ @dataclasses.dataclass(slots=True) class WorkerJob: - test_name: str + runtests: RunTests namespace: Namespace - rerun: bool = False match_tests: FilterTuple | None = None @@ -70,6 +69,7 @@ def default(self, o: Any) -> dict[str, Any]: def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: if "__worker_job__" in d: d.pop('__worker_job__') + d['runtests'] = RunTests(**d['runtests']) return WorkerJob(**d) if "__namespace__" in d: d.pop('__namespace__') @@ -78,17 +78,16 @@ def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: return d -def _parse_worker_args(worker_json: str) -> tuple[Namespace, str]: - return json.loads(worker_json, - object_hook=_decode_worker_job) +def _parse_worker_json(worker_json: str) -> tuple[Namespace, str]: + return json.loads(worker_json, object_hook=_decode_worker_job) -def run_test_in_subprocess(worker_job: WorkerJob, - output_file: TextIO, - tmp_dir: str | None = None) -> subprocess.Popen: +def create_worker_process(worker_job: WorkerJob, + output_file: TextIO, + tmp_dir: str | None = None) -> subprocess.Popen: ns = worker_job.namespace python = ns.python - worker_args = json.dumps(worker_job, cls=_EncodeWorkerJob) + worker_json = json.dumps(worker_job, cls=_EncodeWorkerJob) if python is not None: executable = python @@ -97,7 +96,7 @@ def run_test_in_subprocess(worker_job: WorkerJob, cmd = [*executable, *support.args_from_interpreter_flags(), '-u', # Unbuffered stdout and stderr '-m', 'test.regrtest', - '--worker-args', worker_args] + '--worker-json', worker_json] env = dict(os.environ) if tmp_dir is not None: @@ -122,16 +121,16 @@ def run_test_in_subprocess(worker_job: WorkerJob, return subprocess.Popen(cmd, **kw) -def run_tests_worker(worker_json: str) -> NoReturn: - worker_job = _parse_worker_args(worker_json) +def worker_process(worker_json: str) -> NoReturn: + worker_job = _parse_worker_json(worker_json) + runtests = worker_job.runtests ns = worker_job.namespace - test_name = worker_job.test_name - rerun = worker_job.rerun - match_tests = worker_job.match_tests + test_name = runtests.tests[0] + match_tests: FilterTuple | None = worker_job.match_tests setup_tests(ns) - if rerun: + if runtests.rerun: if match_tests: matching = "matching: " + ", ".join(match_tests) print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) @@ -139,14 +138,15 @@ def run_tests_worker(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) ns.verbose = True - if match_tests is not None: - ns.match_tests = match_tests + if match_tests is not None: + ns.match_tests = match_tests result = runtest(ns, test_name) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON - print(json.dumps(result, cls=EncodeTestResult), flush=True) + json.dump(result, sys.stdout, cls=EncodeTestResult) + sys.stdout.flush() sys.exit(0) @@ -173,7 +173,8 @@ def stop(self): self.tests_iter = None -class MultiprocessResult(NamedTuple): +@dataclasses.dataclass(slots=True, frozen=True) +class MultiprocessResult: result: TestResult # bpo-45410: stderr is written into stdout to keep messages order worker_stdout: str | None = None @@ -198,7 +199,6 @@ def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: self.ns = runner.ns self.timeout = runner.worker_timeout self.regrtest = runner.regrtest - self.rerun = runner.rerun self.current_test_name = None self.start_time = None self._popen = None @@ -264,9 +264,8 @@ def mp_result_error( def _run_process(self, worker_job, output_file: TextIO, tmp_dir: str | None = None) -> int: - self.current_test_name = worker_job.test_name try: - popen = run_test_in_subprocess(worker_job, output_file, tmp_dir) + popen = create_worker_process(worker_job, output_file, tmp_dir) self._killed = False self._popen = popen @@ -316,6 +315,8 @@ def _run_process(self, worker_job, output_file: TextIO, self.current_test_name = None def _runtest(self, test_name: str) -> MultiprocessResult: + self.current_test_name = test_name + if sys.platform == 'win32': # gh-95027: When stdout is not a TTY, Python uses the ANSI code # page for the sys.stdout encoding. If the main process runs in a @@ -324,15 +325,20 @@ def _runtest(self, test_name: str) -> MultiprocessResult: else: encoding = sys.stdout.encoding - match_tests = self.runtests.get_match_tests(test_name) + tests = (test_name,) + if self.runtests.rerun: + match_tests = self.runtests.get_match_tests(test_name) + else: + match_tests = None + worker_runtests = self.runtests.copy(tests=tests) + worker_job = WorkerJob( + worker_runtests, + namespace=self.ns, + match_tests=match_tests) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file: - worker_job = WorkerJob(test_name, - namespace=self.ns, - rerun=self.rerun, - match_tests=match_tests) # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during # Python finalization: too late for libregrtest. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ece8c1f5efdff6..8cced1f5185c2f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -75,10 +75,10 @@ def test_wait(self): ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) - def test_worker_args(self): - ns = libregrtest._parse_args(['--worker-args', '[[], {}]']) - self.assertEqual(ns.worker_args, '[[], {}]') - self.checkError(['--worker-args'], 'expected one argument') + def test_worker_json(self): + ns = libregrtest._parse_args(['--worker-json', '[[], {}]']) + self.assertEqual(ns.worker_json, '[[], {}]') + self.checkError(['--worker-json'], 'expected one argument') def test_start(self): for opt in '-S', '--start': From 057bc7249073066ed8087b548ee06f0eabfa9e7c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 8 Sep 2023 18:24:49 -0700 Subject: [PATCH 580/598] gh-109052: Use the base opcode when comparing code objects (gh-109107) --- Lib/test/test_code.py | 19 +++++++++++++++++++ ...-09-07-18-49-01.gh-issue-109052.TBU4nC.rst | 1 + Objects/codeobject.c | 10 ++++------ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 812c0682569207..a961ddbe17a3d3 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -505,6 +505,25 @@ def test_code_hash_uses_bytecode(self): self.assertNotEqual(c, c1) self.assertNotEqual(hash(c), hash(c1)) + @cpython_only + def test_code_equal_with_instrumentation(self): + """ GH-109052 + + Make sure the instrumentation doesn't affect the code equality + The validity of this test relies on the fact that "x is x" and + "x in x" have only one different instruction and the instructions + have the same argument. + + """ + code1 = compile("x is x", "example.py", "eval") + code2 = compile("x in x", "example.py", "eval") + sys._getframe().f_trace_opcodes = True + sys.settrace(lambda *args: None) + exec(code1, {'x': []}) + exec(code2, {'x': []}) + self.assertNotEqual(code1, code2) + sys.settrace(None) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst new file mode 100644 index 00000000000000..175046c771cdf3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-49-01.gh-issue-109052.TBU4nC.rst @@ -0,0 +1 @@ +Use the base opcode when comparing code objects to avoid interference from instrumentation diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d00bd0422f004d..20e5dedb22826f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1798,28 +1798,26 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - uint8_t co_code = co_instr.op.code; + uint8_t co_code = _Py_GetBaseOpcode(co, i); uint8_t co_arg = co_instr.op.arg; - uint8_t cp_code = cp_instr.op.code; + uint8_t cp_code = _Py_GetBaseOpcode(cp, i); uint8_t cp_arg = cp_instr.op.arg; if (co_code == ENTER_EXECUTOR) { const int exec_index = co_arg; _PyExecutorObject *exec = co->co_executors->executors[exec_index]; - co_code = exec->vm_data.opcode; + co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; co_arg = exec->vm_data.oparg; } assert(co_code != ENTER_EXECUTOR); - co_code = _PyOpcode_Deopt[co_code]; if (cp_code == ENTER_EXECUTOR) { const int exec_index = cp_arg; _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; - cp_code = exec->vm_data.opcode; + cp_code = _PyOpcode_Deopt[exec->vm_data.opcode]; cp_arg = exec->vm_data.oparg; } assert(cp_code != ENTER_EXECUTOR); - cp_code = _PyOpcode_Deopt[cp_code]; if (co_code != cp_code || co_arg != cp_arg) { goto unequal; From e9e2ca7a7b4b4320009cdf85c84ec5bd6c4923c3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 03:37:48 +0200 Subject: [PATCH 581/598] gh-109162: Refactor libregrtest.runtest (#109172) * Rename runtest() to run_single_test(). * Pass runtests to run_single_test(). * Add type annotation to Regrtest attributes. Add missing attributes to Namespace. * Add attributes to Regrtest and RunTests: * fail_fast * ignore_tests * match_tests * output_on_failure * pgo * pgo_extended * timeout * Get pgo from 'runtests', rather than from 'ns'. * Remove WorkerJob.match_tests. * setup_support() now gets pgo_extended from runtests. * save_env(): change parameter order, pass test_name first. * Add setup_test_dir() function. * Pass runtests to setup_tests(). --- Lib/test/libregrtest/cmdline.py | 6 ++ Lib/test/libregrtest/main.py | 91 +++++++++++++++++------------- Lib/test/libregrtest/runtest.py | 60 ++++++++++++-------- Lib/test/libregrtest/runtest_mp.py | 52 ++++++++--------- Lib/test/libregrtest/setup.py | 29 +++++----- 5 files changed, 134 insertions(+), 104 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c08b3be9285d9f..71e7396d71d4a5 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -149,6 +149,10 @@ def __init__(self, **kwargs) -> None: self.verbose = 0 self.quiet = False self.exclude = False + self.cleanup = False + self.wait = False + self.list_cases = False + self.list_tests = False self.single = False self.randomize = False self.fromfile = None @@ -171,6 +175,8 @@ def __init__(self, **kwargs) -> None: self.pgo = False self.pgo_extended = False self.worker_json = None + self.start = None + self.timeout = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cd053c59a98548..f7d28a859213f5 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,10 +11,10 @@ import unittest from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( - findtests, split_test_packages, runtest, abs_module_name, + findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, TestResult, FilterTuple, FilterDict, TestList) -from test.libregrtest.setup import setup_tests +from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, printlist, get_build_info) @@ -64,11 +64,11 @@ def __init__(self, ns: Namespace): self.ns: Namespace = ns # Actions - self.want_header = ns.header - self.want_list_tests = ns.list_tests - self.want_list_cases = ns.list_cases - self.want_wait = ns.wait - self.want_cleanup = ns.cleanup + self.want_header: bool = ns.header + self.want_list_tests: bool = ns.list_tests + self.want_list_cases: bool = ns.list_cases + self.want_wait: bool = ns.wait + self.want_cleanup: bool = ns.cleanup # Select tests if ns.match_tests: @@ -79,14 +79,19 @@ def __init__(self, ns: Namespace): self.ignore_tests: FilterTuple = tuple(ns.ignore_tests) else: self.ignore_tests = None - self.exclude = ns.exclude - self.fromfile = ns.fromfile - self.starting_test = ns.start + self.exclude: bool = ns.exclude + self.fromfile: str | None = ns.fromfile + self.starting_test: str | None = ns.start # Options to run tests - self.forever = ns.forever - self.randomize = ns.randomize - self.random_seed = ns.random_seed + self.fail_fast: bool = ns.failfast + self.forever: bool = ns.forever + self.randomize: bool = ns.randomize + self.random_seed: int | None = ns.random_seed + self.pgo: bool = ns.pgo + self.pgo_extended: bool = ns.pgo_extended + self.output_on_failure: bool = ns.verbose3 + self.timeout: float | None = ns.timeout # tests self.tests = [] @@ -196,21 +201,19 @@ def log(self, line=''): def display_progress(self, test_index, text): quiet = self.ns.quiet - pgo = self.ns.pgo if quiet: return # "[ 51/405/1] test_tcl passed" line = f"{test_index:{self.test_count_width}}{self.test_count_text}" fails = len(self.bad) + len(self.environment_changed) - if fails and not pgo: + if fails and not self.pgo: line = f"{line}/{fails}" self.log(f"[{line}] {text}") def find_tests(self): ns = self.ns single = ns.single - pgo = ns.pgo test_dir = ns.testdir if single: @@ -237,7 +240,7 @@ def find_tests(self): strip_py_suffix(self.tests) - if pgo: + if self.pgo: # add default PGO tests if no tests are specified setup_pgo_tests(ns) @@ -329,8 +332,6 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests ns = self.ns ns.verbose = True - ns.failfast = False - ns.verbose3 = False if ns.use_mp is None: ns.use_mp = 1 @@ -345,12 +346,16 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = runtests.copy(tests=tuple(tests), - match_tests_dict=match_tests_dict, - rerun=True, - forever=False) + runtests = runtests.copy( + tests=tuple(tests), + rerun=True, + forever=False, + fail_fast=False, + match_tests_dict=match_tests_dict, + output_on_failure=False) self.set_tests(runtests) self._run_tests_mp(runtests) + return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): if self.ns.python: @@ -364,16 +369,16 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): self.first_state = self.get_tests_state() print() - self._rerun_failed_tests(need_rerun, runtests) + rerun_runtests = self._rerun_failed_tests(need_rerun, runtests) if self.bad: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - self.display_result() + self.display_result(rerun_runtests) - def display_result(self): - pgo = self.ns.pgo + def display_result(self, runtests): + pgo = runtests.pgo quiet = self.ns.quiet print_slow = self.ns.print_slow @@ -444,12 +449,12 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name)') + cmd = ('result = run_single_test(test_name, runtests, self.ns)') ns = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: - result = runtest(self.ns, test_name) + result = run_single_test(test_name, runtests, self.ns) self.accumulate_result(result) @@ -458,9 +463,7 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): def run_tests_sequentially(self, runtests): ns = self.ns coverage = ns.trace - fail_fast = ns.failfast fail_env_changed = ns.fail_env_changed - timeout = ns.timeout if coverage: import trace @@ -471,8 +474,8 @@ def run_tests_sequentially(self, runtests): save_modules = sys.modules.keys() msg = "Run tests sequentially" - if timeout: - msg += " (timeout: %s)" % format_duration(timeout) + if runtests.timeout: + msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) previous_test = None @@ -492,7 +495,7 @@ def run_tests_sequentially(self, runtests): if module not in save_modules and module.startswith("test."): support.unload(module) - if result.must_stop(fail_fast, fail_env_changed): + if result.must_stop(self.fail_fast, fail_env_changed): break previous_test = str(result) @@ -850,16 +853,28 @@ def action_run_tests(self): # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.ns.pgo or self.ns.quiet or self.ns.single + or not(self.pgo or self.ns.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() if self.randomize: print("Using random seed", self.random_seed) - runtests = RunTests(tuple(self.selected), forever=self.forever) + runtests = RunTests( + tuple(self.selected), + fail_fast=self.fail_fast, + match_tests=self.match_tests, + ignore_tests=self.ignore_tests, + forever=self.forever, + pgo=self.pgo, + pgo_extended=self.pgo_extended, + output_on_failure=self.output_on_failure, + timeout=self.timeout) + + setup_tests(runtests, self.ns) + tracer = self.run_tests(runtests) - self.display_result() + self.display_result(runtests) need_rerun = self.need_rerun if self.ns.rerun and need_rerun: @@ -877,7 +892,7 @@ def _main(self): if self.want_wait: input("Press any key to continue...") - setup_tests(self.ns) + setup_test_dir(self.ns.testdir) self.find_tests() exitcode = 0 diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ea5d0564b14aea..bfb0718aa56c32 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -208,9 +208,16 @@ def get_rerun_match_tests(self) -> FilterTuple | None: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple + fail_fast: bool = False + match_tests: FilterTuple | None = None + ignore_tests: FilterTuple | None = None match_tests_dict: FilterDict | None = None rerun: bool = False forever: bool = False + pgo: bool = False + pgo_extended: bool = False + output_on_failure: bool = False + timeout: float | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -295,11 +302,11 @@ def abs_module_name(test_name: str, test_dir: str | None) -> str: return 'test.' + test_name -def setup_support(ns: Namespace): - support.PGO = ns.pgo - support.PGO_EXTENDED = ns.pgo_extended - support.set_match_tests(ns.match_tests, ns.ignore_tests) - support.failfast = ns.failfast +def setup_support(runtests: RunTests, ns: Namespace): + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + support.failfast = runtests.fail_fast support.verbose = ns.verbose if ns.xmlpath: support.junit_xml_list = [] @@ -307,12 +314,12 @@ def setup_support(ns: Namespace): support.junit_xml_list = None -def _runtest(result: TestResult, ns: Namespace) -> None: +def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. verbose = ns.verbose - output_on_failure = ns.verbose3 - timeout = ns.timeout + output_on_failure = runtests.output_on_failure + timeout = runtests.timeout use_timeout = ( timeout is not None and threading_helper.can_start_thread @@ -321,7 +328,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: faulthandler.dump_traceback_later(timeout, exit=True) try: - setup_support(ns) + setup_support(runtests, ns) if output_on_failure: support.verbose = True @@ -341,7 +348,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: # warnings will be written to sys.stderr below. print_warning.orig_stderr = stream - _runtest_env_changed_exc(result, ns, display_failure=False) + _runtest_env_changed_exc(result, runtests, ns, display_failure=False) # Ignore output if the test passed successfully if result.state != State.PASSED: output = stream.getvalue() @@ -356,7 +363,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, ns, display_failure=not verbose) + _runtest_env_changed_exc(result, runtests, ns, display_failure=not verbose) xml_list = support.junit_xml_list if xml_list: @@ -369,7 +376,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: support.junit_xml_list = None -def runtest(ns: Namespace, test_name: str) -> TestResult: +def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestResult: """Run a single test. ns -- regrtest namespace of options @@ -382,10 +389,11 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: """ start_time = time.perf_counter() result = TestResult(test_name) + pgo = runtests.pgo try: - _runtest(result, ns) + _runtest(result, runtests, ns) except: - if not ns.pgo: + if not pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) @@ -404,8 +412,8 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(ns: Namespace, test_name: str): - return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) +def save_env(test_name: str, runtests: RunTests, ns: Namespace): + return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=runtests.pgo) def regrtest_runner(result, test_func, ns) -> None: @@ -442,7 +450,7 @@ def regrtest_runner(result, test_func, ns) -> None: FOUND_GARBAGE = [] -def _load_run_test(result: TestResult, ns: Namespace) -> None: +def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Load the test function, run the test function. module_name = abs_module_name(result.test_name, ns.testdir) @@ -458,7 +466,7 @@ def test_func(): return run_unittest(test_mod) try: - with save_env(ns, result.test_name): + with save_env(result.test_name, runtests, ns): regrtest_runner(result, test_func, ns) finally: # First kill any dangling references to open files etc. @@ -482,7 +490,8 @@ def test_func(): support.reap_children() -def _runtest_env_changed_exc(result: TestResult, ns: Namespace, +def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, + ns: Namespace, display_failure: bool = True) -> None: # Detect environment changes, handle exceptions. @@ -490,7 +499,8 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, # the environment support.environment_altered = False - if ns.pgo: + pgo = runtests.pgo + if pgo: display_failure = False test_name = result.test_name @@ -498,15 +508,15 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, clear_caches() support.gc_collect() - with save_env(ns, test_name): - _load_run_test(result, ns) + with save_env(test_name, runtests, ns): + _load_run_test(result, runtests, ns) except support.ResourceDenied as msg: - if not ns.quiet and not ns.pgo: + if not ns.quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as msg: - if not ns.quiet and not ns.pgo: + if not ns.quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.SKIPPED return @@ -536,7 +546,7 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, result.state = State.INTERRUPTED return except: - if not ns.pgo: + if not pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 45db24f255fcf7..ecdde3aa523098 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -10,7 +10,7 @@ import threading import time import traceback -from typing import NamedTuple, NoReturn, Literal, Any, TextIO +from typing import NoReturn, Literal, Any, TextIO from test import support from test.support import os_helper @@ -19,9 +19,9 @@ from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - runtest, TestResult, State, PROGRESS_MIN_TIME, + run_single_test, TestResult, State, PROGRESS_MIN_TIME, FilterTuple, RunTests) -from test.libregrtest.setup import setup_tests +from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.utils import format_duration, print_warning if sys.platform == 'win32': @@ -48,7 +48,6 @@ class WorkerJob: runtests: RunTests namespace: Namespace - match_tests: FilterTuple | None = None class _EncodeWorkerJob(json.JSONEncoder): @@ -126,9 +125,10 @@ def worker_process(worker_json: str) -> NoReturn: runtests = worker_job.runtests ns = worker_job.namespace test_name = runtests.tests[0] - match_tests: FilterTuple | None = worker_job.match_tests + match_tests: FilterTuple | None = runtests.match_tests - setup_tests(ns) + setup_test_dir(ns.testdir) + setup_tests(runtests, ns) if runtests.rerun: if match_tests: @@ -138,10 +138,7 @@ def worker_process(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) ns.verbose = True - if match_tests is not None: - ns.match_tests = match_tests - - result = runtest(ns, test_name) + result = run_single_test(test_name, runtests, ns) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON @@ -330,11 +327,13 @@ def _runtest(self, test_name: str) -> MultiprocessResult: match_tests = self.runtests.get_match_tests(test_name) else: match_tests = None - worker_runtests = self.runtests.copy(tests=tests) + kwargs = {} + if match_tests: + kwargs['match_tests'] = match_tests + worker_runtests = self.runtests.copy(tests=tests, **kwargs) worker_job = WorkerJob( worker_runtests, - namespace=self.ns, - match_tests=match_tests) + namespace=self.ns) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. @@ -401,7 +400,7 @@ def _runtest(self, test_name: str) -> MultiprocessResult: return MultiprocessResult(result, stdout) def run(self) -> None: - fail_fast = self.ns.failfast + fail_fast = self.runtests.fail_fast fail_env_changed = self.ns.fail_env_changed while not self._stopped: try: @@ -473,7 +472,6 @@ def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: class MultiprocessTestRunner: def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: ns = regrtest.ns - timeout = ns.timeout self.regrtest = regrtest self.runtests = runtests @@ -483,24 +481,24 @@ def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: self.output: queue.Queue[QueueOutput] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) - if timeout is not None: + self.timeout = runtests.timeout + if self.timeout is not None: # Rely on faulthandler to kill a worker process. This timouet is # when faulthandler fails to kill a worker process. Give a maximum # of 5 minutes to faulthandler to kill the worker. - self.worker_timeout = min(timeout * 1.5, timeout + 5 * 60) + self.worker_timeout = min(self.timeout * 1.5, self.timeout + 5 * 60) else: self.worker_timeout = None self.workers = None def start_workers(self) -> None: use_mp = self.ns.use_mp - timeout = self.ns.timeout self.workers = [TestWorkerProcess(index, self) for index in range(1, use_mp + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" - if timeout: + if self.timeout: msg += (" (timeout: %s, worker timeout: %s)" - % (format_duration(timeout), + % (format_duration(self.timeout), format_duration(self.worker_timeout))) self.log(msg) for worker in self.workers: @@ -514,9 +512,8 @@ def stop_workers(self) -> None: worker.wait_stopped(start_time) def _get_result(self) -> QueueOutput | None: - pgo = self.ns.pgo - use_faulthandler = (self.ns.timeout is not None) - timeout = PROGRESS_UPDATE + pgo = self.runtests.pgo + use_faulthandler = (self.timeout is not None) # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. @@ -527,7 +524,7 @@ def _get_result(self) -> QueueOutput | None: # wait for a thread try: - return self.output.get(timeout=timeout) + return self.output.get(timeout=PROGRESS_UPDATE) except queue.Empty: pass @@ -544,7 +541,7 @@ def _get_result(self) -> QueueOutput | None: def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result - pgo = self.ns.pgo + pgo = self.runtests.pgo text = str(result) if mp_result.err_msg: @@ -580,9 +577,8 @@ def _process_result(self, item: QueueOutput) -> bool: return result def run_tests(self) -> None: - fail_fast = self.ns.failfast + fail_fast = self.runtests.fail_fast fail_env_changed = self.ns.fail_env_changed - timeout = self.ns.timeout self.start_workers() @@ -600,7 +596,7 @@ def run_tests(self) -> None: print() self.regrtest.interrupted = True finally: - if timeout is not None: + if self.timeout is not None: faulthandler.cancel_dump_traceback_later() # Always ensure that all worker processes are no longer diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index b76bece7ca08b5..f640362cb2df7f 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -18,7 +18,14 @@ UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" -def setup_tests(ns): +def setup_test_dir(testdir): + if testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(testdir)) + + +def setup_tests(runtests, ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -44,11 +51,6 @@ def setup_tests(ns): replace_stdout() support.record_original_stdout(sys.stdout) - if ns.testdir: - # Prepend test directory to sys.path, so runtest() will be able - # to locate tests - sys.path.insert(0, os.path.abspath(ns.testdir)) - # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). @@ -88,16 +90,17 @@ def _test_audit_hook(name, args): setup_unraisable_hook() setup_threading_excepthook() - if ns.timeout is not None: + timeout = runtests.timeout + if timeout is not None: # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT - support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, ns.timeout / 40) - support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, ns.timeout / 4) + support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, timeout / 40) + support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, timeout / 4) # If --timeout is short: reduce timeouts - support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, ns.timeout) - support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, ns.timeout) - support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, ns.timeout) - support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, ns.timeout) + support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, timeout) + support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, timeout) + support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) + support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) if ns.xmlpath: from test.support.testresult import RegressionTestResult From 17f994174de9211b2baaff217eeb1033343230fc Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 8 Sep 2023 19:49:20 -0700 Subject: [PATCH 582/598] gh-109118: Fix runtime crash when NameError happens in PEP 695 function (#109123) --- Include/internal/pycore_opcode_metadata.h | 90 ++++++++----------- Lib/test/test_type_params.py | 40 +++++++++ ...-09-07-18-24-42.gh-issue-109118.yPXRAe.rst | 2 + Python/abstract_interp_cases.c.h | 10 ++- Python/bytecodes.c | 44 +++++++-- Python/executor_cases.c.h | 42 ++++++++- Python/generated_cases.c.h | 67 ++++++-------- 7 files changed, 189 insertions(+), 106 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fb5c0465a10021..f70b75a56c150c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -39,40 +39,38 @@ #define _BINARY_OP_ADD_UNICODE 311 #define _BINARY_OP_INPLACE_ADD_UNICODE 312 #define _POP_FRAME 313 -#define _LOAD_LOCALS 314 -#define _LOAD_FROM_DICT_OR_GLOBALS 315 -#define _GUARD_GLOBALS_VERSION 316 -#define _GUARD_BUILTINS_VERSION 317 -#define _LOAD_GLOBAL_MODULE 318 -#define _LOAD_GLOBAL_BUILTINS 319 -#define _GUARD_TYPE_VERSION 320 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 321 -#define _LOAD_ATTR_INSTANCE_VALUE 322 -#define IS_NONE 323 -#define _ITER_CHECK_LIST 324 -#define _ITER_JUMP_LIST 325 -#define _IS_ITER_EXHAUSTED_LIST 326 -#define _ITER_NEXT_LIST 327 -#define _ITER_CHECK_TUPLE 328 -#define _ITER_JUMP_TUPLE 329 -#define _IS_ITER_EXHAUSTED_TUPLE 330 -#define _ITER_NEXT_TUPLE 331 -#define _ITER_CHECK_RANGE 332 -#define _ITER_JUMP_RANGE 333 -#define _IS_ITER_EXHAUSTED_RANGE 334 -#define _ITER_NEXT_RANGE 335 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 336 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 337 -#define _CHECK_PEP_523 338 -#define _CHECK_FUNCTION_EXACT_ARGS 339 -#define _CHECK_STACK_SPACE 340 -#define _INIT_CALL_PY_EXACT_ARGS 341 -#define _PUSH_FRAME 342 -#define _POP_JUMP_IF_FALSE 343 -#define _POP_JUMP_IF_TRUE 344 -#define JUMP_TO_TOP 345 -#define SAVE_CURRENT_IP 346 -#define INSERT 347 +#define _GUARD_GLOBALS_VERSION 314 +#define _GUARD_BUILTINS_VERSION 315 +#define _LOAD_GLOBAL_MODULE 316 +#define _LOAD_GLOBAL_BUILTINS 317 +#define _GUARD_TYPE_VERSION 318 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 +#define _LOAD_ATTR_INSTANCE_VALUE 320 +#define IS_NONE 321 +#define _ITER_CHECK_LIST 322 +#define _ITER_JUMP_LIST 323 +#define _IS_ITER_EXHAUSTED_LIST 324 +#define _ITER_NEXT_LIST 325 +#define _ITER_CHECK_TUPLE 326 +#define _ITER_JUMP_TUPLE 327 +#define _IS_ITER_EXHAUSTED_TUPLE 328 +#define _ITER_NEXT_TUPLE 329 +#define _ITER_CHECK_RANGE 330 +#define _ITER_JUMP_RANGE 331 +#define _IS_ITER_EXHAUSTED_RANGE 332 +#define _ITER_NEXT_RANGE 333 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 334 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 335 +#define _CHECK_PEP_523 336 +#define _CHECK_FUNCTION_EXACT_ARGS 337 +#define _CHECK_STACK_SPACE 338 +#define _INIT_CALL_PY_EXACT_ARGS 339 +#define _PUSH_FRAME 340 +#define _POP_JUMP_IF_FALSE 341 +#define _POP_JUMP_IF_TRUE 342 +#define JUMP_TO_TOP 343 +#define SAVE_CURRENT_IP 344 +#define INSERT 345 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -268,16 +266,12 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case DELETE_GLOBAL: return 0; - case _LOAD_LOCALS: - return 0; case LOAD_LOCALS: return 0; - case _LOAD_FROM_DICT_OR_GLOBALS: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_NAME: return 0; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; case LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: @@ -802,16 +796,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case DELETE_GLOBAL: return 0; - case _LOAD_LOCALS: - return 1; case LOAD_LOCALS: return 1; - case _LOAD_FROM_DICT_OR_GLOBALS: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_NAME: return 1; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; case _GUARD_GLOBALS_VERSION: @@ -1305,11 +1295,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [_LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [_LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, @@ -1546,9 +1534,9 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, - [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, @@ -1633,8 +1621,6 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_POP_FRAME] = "_POP_FRAME", - [_LOAD_LOCALS] = "_LOAD_LOCALS", - [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 0045057f181e1c..f93d088ea758a9 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -956,3 +956,43 @@ class NewStyle[T]: for case in cases: with self.subTest(case=case): weakref.ref(case) + + +class TypeParamsRuntimeTest(unittest.TestCase): + def test_name_error(self): + # gh-109118: This crashed the interpreter due to a refcounting bug + code = """ + class name_2[name_5]: + class name_4[name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + # Crashed with a slightly different stack trace + code = """ + class name_2[name_5]: + class name_4[name_5: name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + def test_broken_class_namespace(self): + code = """ + class WeirdMapping(dict): + def __missing__(self, key): + if key == "T": + raise RuntimeError + raise KeyError(key) + + class Meta(type): + def __prepare__(name, bases): + return WeirdMapping() + + class MyClass[V](metaclass=Meta): + class Inner[U](T): + pass + """ + with self.assertRaises(RuntimeError): + run_code(code) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst new file mode 100644 index 00000000000000..f14fce4423896f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-07-18-24-42.gh-issue-109118.yPXRAe.rst @@ -0,0 +1,2 @@ +Fix interpreter crash when a NameError is raised inside the type parameters +of a generic class. diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 11fbf4443dda16..398c04616bc091 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -291,13 +291,19 @@ break; } - case _LOAD_LOCALS: { + case LOAD_LOCALS: { STACK_GROW(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { + case LOAD_FROM_DICT_OR_GLOBALS: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_NAME: { + STACK_GROW(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 21069204cf8693..ff95f37b900da6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1286,7 +1286,7 @@ dummy_func( } } - op(_LOAD_LOCALS, ( -- locals)) { + inst(LOAD_LOCALS, ( -- locals)) { locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1296,15 +1296,11 @@ dummy_func( Py_INCREF(locals); } - macro(LOAD_LOCALS) = _LOAD_LOCALS; - - op(_LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { + inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); goto error; } - Py_DECREF(mod_or_class_dict); if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); if (v != NULL) { @@ -1325,11 +1321,41 @@ dummy_func( } } } + DECREF_INPUTS(); } - macro(LOAD_NAME) = _LOAD_LOCALS + _LOAD_FROM_DICT_OR_GLOBALS; - - macro(LOAD_FROM_DICT_OR_GLOBALS) = _LOAD_FROM_DICT_OR_GLOBALS; + inst(LOAD_NAME, (-- v)) { + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + ERROR_IF(true, error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; + } + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + goto error; + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } + } + } + } family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { LOAD_GLOBAL_MODULE, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fa7cb88e4a1e1f..918991dca7dd25 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1046,7 +1046,7 @@ break; } - case _LOAD_LOCALS: { + case LOAD_LOCALS: { PyObject *locals; locals = LOCALS(); if (locals == NULL) { @@ -1060,16 +1060,51 @@ break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { + case LOAD_FROM_DICT_OR_GLOBALS: { PyObject *mod_or_class_dict; PyObject *v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); goto error; } + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + goto error; + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } + } + } Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + break; + } + + case LOAD_NAME: { + PyObject *v; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; + } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); if (v != NULL) { @@ -1090,6 +1125,7 @@ } } } + STACK_GROW(1); stack_pointer[-1] = v; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 136b36c8260cb5..e84599d87a6968 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1675,65 +1675,51 @@ DISPATCH(); } - TARGET(LOAD_NAME) { - PyObject *locals; + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { PyObject *mod_or_class_dict; PyObject *v; - // _LOAD_LOCALS - { - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + goto error; } - // _LOAD_FROM_DICT_OR_GLOBALS - mod_or_class_dict = locals; - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { goto error; } - Py_DECREF(mod_or_class_dict); - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { goto error; } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; - } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; } } } - STACK_GROW(1); + Py_DECREF(mod_or_class_dict); stack_pointer[-1] = v; DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *mod_or_class_dict; + TARGET(LOAD_NAME) { PyObject *v; - mod_or_class_dict = stack_pointer[-1]; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); goto error; } - Py_DECREF(mod_or_class_dict); if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); if (v != NULL) { @@ -1754,6 +1740,7 @@ } } } + STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); } From b4131a13cb41d0f397776683c3b99500db9e2cfd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 9 Sep 2023 08:44:46 +0300 Subject: [PATCH 583/598] gh-109050: Remove remaining tests for legacy Unicode C API (GH-109068) --- Lib/test/support/__init__.py | 9 ----- Lib/test/test_capi/test_getargs.py | 64 ------------------------------ Lib/test/test_csv.py | 12 ------ Lib/test/test_decimal.py | 31 +-------------- Lib/test/test_str.py | 30 -------------- Modules/_testcapi/getargs.c | 52 ------------------------ 6 files changed, 1 insertion(+), 197 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 84b74ee2c298e9..f40b73b0a799f1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -506,15 +506,6 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) -def requires_legacy_unicode_capi(): - try: - from _testcapi import unicode_legacy_string - except ImportError: - unicode_legacy_string = None - - return unittest.skipUnless(unicode_legacy_string, - 'requires legacy Unicode C API') - # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 246206af86101c..e10f679eeb71c8 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1004,70 +1004,6 @@ def test_et_hash(self): buf = bytearray() self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) - @support.requires_legacy_unicode_capi() - def test_u(self): - from _testcapi import getargs_u - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_u('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, getargs_u, 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u, None) - - @support.requires_legacy_unicode_capi() - def test_u_hash(self): - from _testcapi import getargs_u_hash - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_u_hash('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_u_hash('nul:\0'), 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_u_hash, None) - - @support.requires_legacy_unicode_capi() - def test_Z(self): - from _testcapi import getargs_Z - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_Z('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, getargs_Z, 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertIsNone(getargs_Z(None)) - - @support.requires_legacy_unicode_capi() - def test_Z_hash(self): - from _testcapi import getargs_Z_hash - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_Z_hash('abc\xe9'), 'abc\xe9') - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_Z_hash('nul:\0'), 'nul:\0') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z_hash, b'bytes') - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z_hash, bytearray(b'bytearray')) - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, getargs_Z_hash, memoryview(b'memoryview')) - with self.assertWarns(DeprecationWarning): - self.assertIsNone(getargs_Z_hash(None)) - def test_gh_99240_clear_args(self): from _testcapi import gh_99240_clear_args self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index bc6879176cd85e..27f4978ca66a88 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -281,18 +281,6 @@ def test_writerows_errors(self): self.assertRaises(TypeError, writer.writerows, None) self.assertRaises(OSError, writer.writerows, BadIterable()) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_writerows_legacy_strings(self): - import _testcapi - c = _testcapi.unicode_legacy_string('a') - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - writer.writerows([[c]]) - fileobj.seek(0) - self.assertEqual(fileobj.read(), "a\r\n") - def _read_test(self, input, expect, **kwargs): reader = csv.reader(input, **kwargs) result = list(reader) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index abfd71c868d009..d806eeac25fb2f 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -34,7 +34,7 @@ import locale from test.support import (is_resource_enabled, requires_IEEE_754, requires_docstrings, - requires_legacy_unicode_capi, check_sanitizer, + check_sanitizer, check_disallow_instantiation) from test.support import (TestFailed, run_with_locale, cpython_only, @@ -587,18 +587,6 @@ def test_explicit_from_string(self): # underscores don't prevent errors self.assertRaises(InvalidOperation, Decimal, "1_2_\u00003") - @cpython_only - @requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_from_legacy_strings(self): - import _testcapi - Decimal = self.decimal.Decimal - context = self.decimal.Context() - - s = _testcapi.unicode_legacy_string('9.999999') - self.assertEqual(str(Decimal(s)), '9.999999') - self.assertEqual(str(context.create_decimal(s)), '9.999999') - def test_explicit_from_tuples(self): Decimal = self.decimal.Decimal @@ -2919,23 +2907,6 @@ def test_none_args(self): assert_signals(self, c, 'traps', [InvalidOperation, DivisionByZero, Overflow]) - @cpython_only - @requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_from_legacy_strings(self): - import _testcapi - c = self.decimal.Context() - - for rnd in RoundingModes: - c.rounding = _testcapi.unicode_legacy_string(rnd) - self.assertEqual(c.rounding, rnd) - - s = _testcapi.unicode_legacy_string('') - self.assertRaises(TypeError, setattr, c, 'rounding', s) - - s = _testcapi.unicode_legacy_string('ROUND_\x00UP') - self.assertRaises(TypeError, setattr, c, 'rounding', s) - def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 3ae2f45ef6bddc..814ef111c5bec8 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -812,16 +812,6 @@ def test_isidentifier(self): self.assertFalse("©".isidentifier()) self.assertFalse("0".isidentifier()) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_isidentifier_legacy(self): - u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊' - self.assertTrue(u.isidentifier()) - with warnings_helper.check_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier()) - def test_isprintable(self): self.assertTrue("".isprintable()) self.assertTrue(" ".isprintable()) @@ -2489,26 +2479,6 @@ def test_getnewargs(self): self.assertEqual(args[0], text) self.assertEqual(len(args), 1) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_resize(self): - for length in range(1, 100, 7): - # generate a fresh string (refcount=1) - text = 'a' * length + 'b' - - # fill wstr internal field - with self.assertWarns(DeprecationWarning): - abc = _testcapi.getargs_u(text) - self.assertEqual(abc, text) - - # resize text: wstr field must be cleared and then recomputed - text += 'c' - with self.assertWarns(DeprecationWarning): - abcdef = _testcapi.getargs_u(text) - self.assertNotEqual(abc, abcdef) - self.assertEqual(abcdef, text) - def test_compare(self): # Issue #17615 N = 10 diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 10a1c1dd05253d..5f4a6dc8ca7672 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -589,54 +589,6 @@ getargs_y_hash(PyObject *self, PyObject *args) return PyBytes_FromStringAndSize(str, size); } -static PyObject * -getargs_u(PyObject *self, PyObject *args) -{ - wchar_t *str; - if (!PyArg_ParseTuple(args, "u", &str)) { - return NULL; - } - return PyUnicode_FromWideChar(str, -1); -} - -static PyObject * -getargs_u_hash(PyObject *self, PyObject *args) -{ - wchar_t *str; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "u#", &str, &size)) { - return NULL; - } - return PyUnicode_FromWideChar(str, size); -} - -static PyObject * -getargs_Z(PyObject *self, PyObject *args) -{ - wchar_t *str; - if (!PyArg_ParseTuple(args, "Z", &str)) { - return NULL; - } - if (str != NULL) { - return PyUnicode_FromWideChar(str, -1); - } - Py_RETURN_NONE; -} - -static PyObject * -getargs_Z_hash(PyObject *self, PyObject *args) -{ - wchar_t *str; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Z#", &str, &size)) { - return NULL; - } - if (str != NULL) { - return PyUnicode_FromWideChar(str, size); - } - Py_RETURN_NONE; -} - static PyObject * getargs_es(PyObject *self, PyObject *args) { @@ -845,8 +797,6 @@ static PyMethodDef test_methods[] = { {"getargs_S", getargs_S, METH_VARARGS}, {"getargs_U", getargs_U, METH_VARARGS}, {"getargs_Y", getargs_Y, METH_VARARGS}, - {"getargs_Z", getargs_Z, METH_VARARGS}, - {"getargs_Z_hash", getargs_Z_hash, METH_VARARGS}, {"getargs_b", getargs_b, METH_VARARGS}, {"getargs_c", getargs_c, METH_VARARGS}, {"getargs_d", getargs_d, METH_VARARGS}, @@ -868,8 +818,6 @@ static PyMethodDef test_methods[] = { {"getargs_s_hash", getargs_s_hash, METH_VARARGS}, {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, - {"getargs_u", getargs_u, METH_VARARGS}, - {"getargs_u_hash", getargs_u_hash, METH_VARARGS}, {"getargs_w_star", getargs_w_star, METH_VARARGS}, {"getargs_y", getargs_y, METH_VARARGS}, {"getargs_y_hash", getargs_y_hash, METH_VARARGS}, From e21c89f9841488a1d4ca1024dc97c1aba44e0c87 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 9 Sep 2023 11:18:14 +0200 Subject: [PATCH 584/598] gh-109162: Refactor libregrtest.RunTests (#109177) * Rename dash_R() runtest_refleak(). The function now gets huntrleaks and quiet arguments, instead of 'ns' argument. * Add attributes to Regrtest and RunTests: * verbose * quiet * huntrleaks * test_dir * Add HuntRefleak class. --- Lib/test/libregrtest/main.py | 57 +++++++++++++++++------------- Lib/test/libregrtest/refleak.py | 23 +++++++----- Lib/test/libregrtest/runtest.py | 57 +++++++++++++++++++----------- Lib/test/libregrtest/runtest_mp.py | 3 +- Lib/test/libregrtest/setup.py | 8 ++--- 5 files changed, 88 insertions(+), 60 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f7d28a859213f5..bb02101d97a2d2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,7 +12,7 @@ from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, - PROGRESS_MIN_TIME, State, RunTests, TestResult, + PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak, FilterTuple, FilterDict, TestList) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests @@ -92,6 +92,14 @@ def __init__(self, ns: Namespace): self.pgo_extended: bool = ns.pgo_extended self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout + self.verbose: bool = ns.verbose + self.quiet: bool = ns.quiet + if ns.huntrleaks: + self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks) + else: + self.hunt_refleak = None + self.test_dir: str | None = ns.testdir + self.junit_filename: str | None = ns.xmlpath # tests self.tests = [] @@ -200,8 +208,7 @@ def log(self, line=''): print(line, flush=True) def display_progress(self, test_index, text): - quiet = self.ns.quiet - if quiet: + if self.quiet: return # "[ 51/405/1] test_tcl passed" @@ -214,7 +221,6 @@ def display_progress(self, test_index, text): def find_tests(self): ns = self.ns single = ns.single - test_dir = ns.testdir if single: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') @@ -250,7 +256,8 @@ def find_tests(self): exclude_tests.add(arg) ns.args = [] - alltests = findtests(testdir=test_dir, exclude=exclude_tests) + alltests = findtests(testdir=self.test_dir, + exclude=exclude_tests) if not self.fromfile: self.selected = self.tests or ns.args @@ -298,14 +305,12 @@ def _list_cases(self, suite): print(test.id()) def list_cases(self): - ns = self.ns - test_dir = ns.testdir support.verbose = False support.set_match_tests(self.match_tests, self.ignore_tests) skipped = [] for test_name in self.selected: - module_name = abs_module_name(test_name, test_dir) + module_name = abs_module_name(test_name, self.test_dir) try: suite = unittest.defaultTestLoader.loadTestsFromName(module_name) self._list_cases(suite) @@ -331,7 +336,6 @@ def get_rerun_match(self, rerun_list) -> FilterDict: def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests ns = self.ns - ns.verbose = True if ns.use_mp is None: ns.use_mp = 1 @@ -349,6 +353,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): runtests = runtests.copy( tests=tuple(tests), rerun=True, + verbose=True, forever=False, fail_fast=False, match_tests_dict=match_tests_dict, @@ -379,7 +384,6 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): def display_result(self, runtests): pgo = runtests.pgo - quiet = self.ns.quiet print_slow = self.ns.print_slow # If running the test suite for PGO then no one cares about results. @@ -398,7 +402,7 @@ def display_result(self, runtests): print(count(len(omitted), "test"), "omitted:") printlist(omitted) - if self.good and not quiet: + if self.good and not self.quiet: print() if (not self.bad and not self.skipped @@ -425,12 +429,12 @@ def display_result(self, runtests): count(len(self.environment_changed), "test"))) printlist(self.environment_changed) - if self.skipped and not quiet: + if self.skipped and not self.quiet: print() print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) - if self.resource_denied and not quiet: + if self.resource_denied and not self.quiet: print() print(count(len(self.resource_denied), "test"), "skipped (resource denied):") printlist(self.resource_denied) @@ -684,7 +688,7 @@ def display_summary(self): print(f"Result: {result}") def save_xml_result(self): - if not self.ns.xmlpath and not self.testsuite_xml: + if not self.junit_filename and not self.testsuite_xml: return import xml.etree.ElementTree as ET @@ -703,7 +707,7 @@ def save_xml_result(self): for k, v in totals.items(): root.set(k, str(v)) - xmlpath = os.path.join(os_helper.SAVEDCWD, self.ns.xmlpath) + xmlpath = os.path.join(os_helper.SAVEDCWD, self.junit_filename) with open(xmlpath, 'wb') as f: for s in ET.tostringlist(root): f.write(s) @@ -785,7 +789,7 @@ def main(self, tests: TestList | None = None): ns = self.ns self.tests = tests - if ns.xmlpath: + if self.junit_filename: support.junit_xml_list = self.testsuite_xml = [] strip_py_suffix(ns.args) @@ -844,16 +848,14 @@ def get_exitcode(self): return exitcode def action_run_tests(self): - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and less than " - "3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.pgo or self.ns.quiet or self.ns.single + or not(self.pgo or self.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() @@ -869,7 +871,12 @@ def action_run_tests(self): pgo=self.pgo, pgo_extended=self.pgo_extended, output_on_failure=self.output_on_failure, - timeout=self.timeout) + timeout=self.timeout, + verbose=self.verbose, + quiet=self.quiet, + hunt_refleak=self.hunt_refleak, + test_dir=self.test_dir, + junit_filename=self.junit_filename) setup_tests(runtests, self.ns) @@ -892,7 +899,7 @@ def _main(self): if self.want_wait: input("Press any key to continue...") - setup_test_dir(self.ns.testdir) + setup_test_dir(self.test_dir) self.find_tests() exitcode = 0 diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 206802b60ddcd0..2e9f17e1c1eee6 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -4,6 +4,7 @@ from inspect import isabstract from test import support from test.support import os_helper +from test.libregrtest.runtest import HuntRefleak from test.libregrtest.utils import clear_caches try: @@ -19,7 +20,9 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) -def dash_R(ns, test_name, test_func): +def runtest_refleak(test_name, test_func, + hunt_refleak: HuntRefleak, + quiet: bool): """Run a test multiple times, looking for reference leaks. Returns: @@ -62,9 +65,11 @@ def dash_R(ns, test_name, test_func): def get_pooled_int(value): return int_pool.setdefault(value, value) - nwarmup, ntracked, fname = ns.huntrleaks - fname = os.path.join(os_helper.SAVEDCWD, fname) - repcount = nwarmup + ntracked + warmups = hunt_refleak.warmups + runs = hunt_refleak.runs + filename = hunt_refleak.filename + filename = os.path.join(os_helper.SAVEDCWD, filename) + repcount = warmups + runs # Pre-allocate to ensure that the loop doesn't allocate anything new rep_range = list(range(repcount)) @@ -78,7 +83,7 @@ def get_pooled_int(value): # initialize variables to make pyflakes quiet rc_before = alloc_before = fd_before = interned_before = 0 - if not ns.quiet: + if not quiet: print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) @@ -102,7 +107,7 @@ def get_pooled_int(value): rc_after = gettotalrefcount() - interned_after * 2 fd_after = fd_count() - if not ns.quiet: + if not quiet: print('.', end='', file=sys.stderr, flush=True) rc_deltas[i] = get_pooled_int(rc_after - rc_before) @@ -114,7 +119,7 @@ def get_pooled_int(value): fd_before = fd_after interned_before = interned_after - if not ns.quiet: + if not quiet: print(file=sys.stderr) # These checkers return False on success, True on failure @@ -143,12 +148,12 @@ def check_fd_deltas(deltas): (fd_deltas, 'file descriptors', check_fd_deltas) ]: # ignore warmup runs - deltas = deltas[nwarmup:] + deltas = deltas[warmups:] if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) print(msg, file=sys.stderr, flush=True) - with open(fname, "a", encoding="utf-8") as refrep: + with open(filename, "a", encoding="utf-8") as refrep: print(msg, file=refrep) refrep.flush() failed = True diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index bfb0718aa56c32..4f12176ced5c87 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -28,6 +28,13 @@ FilterDict = dict[str, FilterTuple] +@dataclasses.dataclass(slots=True, frozen=True) +class HuntRefleak: + warmups: int + runs: int + filename: str + + # Avoid enum.Enum to reduce the number of imports when tests are run class State: PASSED = "PASSED" @@ -218,6 +225,11 @@ class RunTests: pgo_extended: bool = False output_on_failure: bool = False timeout: float | None = None + verbose: bool = False + quiet: bool = False + hunt_refleak: HuntRefleak | None = None + test_dir: str | None = None + junit_filename: str | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -260,7 +272,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(*, testdir=None, exclude=(), +def findtests(*, testdir: str | None =None, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) @@ -279,7 +291,7 @@ def findtests(*, testdir=None, exclude=(), return sorted(tests) -def split_test_packages(tests, *, testdir=None, exclude=(), +def split_test_packages(tests, *, testdir: str | None = None, exclude=(), split_test_dirs=SPLITTESTDIRS): testdir = findtestdir(testdir) splitted = [] @@ -307,8 +319,8 @@ def setup_support(runtests: RunTests, ns: Namespace): support.PGO_EXTENDED = runtests.pgo_extended support.set_match_tests(runtests.match_tests, runtests.ignore_tests) support.failfast = runtests.fail_fast - support.verbose = ns.verbose - if ns.xmlpath: + support.verbose = runtests.verbose + if runtests.junit_filename: support.junit_xml_list = [] else: support.junit_xml_list = None @@ -317,7 +329,7 @@ def setup_support(runtests: RunTests, ns: Namespace): def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. - verbose = ns.verbose + verbose = runtests.verbose output_on_failure = runtests.output_on_failure timeout = runtests.timeout @@ -363,7 +375,8 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, runtests, ns, display_failure=not verbose) + _runtest_env_changed_exc(result, runtests, ns, + display_failure=not verbose) xml_list = support.junit_xml_list if xml_list: @@ -384,7 +397,7 @@ def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestRe Returns a TestResult. - If ns.xmlpath is not None, xml_data is a list containing each + If runtests.junit_filename is not None, xml_data is a list containing each generated testsuite element. """ start_time = time.perf_counter() @@ -412,16 +425,19 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(test_name: str, runtests: RunTests, ns: Namespace): - return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=runtests.pgo) +def save_env(test_name: str, runtests: RunTests): + return saved_test_environment(test_name, runtests.verbose, runtests.quiet, + pgo=runtests.pgo) -def regrtest_runner(result, test_func, ns) -> None: +def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Run test_func(), collect statistics, and detect reference and memory # leaks. - if ns.huntrleaks: - from test.libregrtest.refleak import dash_R - refleak, test_result = dash_R(ns, result.test_name, test_func) + if runtests.hunt_refleak: + from test.libregrtest.refleak import runtest_refleak + refleak, test_result = runtest_refleak(result.test_name, test_func, + runtests.hunt_refleak, + runtests.quiet) else: test_result = test_func() refleak = False @@ -452,7 +468,7 @@ def regrtest_runner(result, test_func, ns) -> None: def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Load the test function, run the test function. - module_name = abs_module_name(result.test_name, ns.testdir) + module_name = abs_module_name(result.test_name, runtests.test_dir) # Remove the module from sys.module to reload it if it was already imported sys.modules.pop(module_name, None) @@ -466,8 +482,8 @@ def test_func(): return run_unittest(test_mod) try: - with save_env(result.test_name, runtests, ns): - regrtest_runner(result, test_func, ns) + with save_env(result.test_name, runtests): + regrtest_runner(result, test_func, runtests) finally: # First kill any dangling references to open files etc. # This can also issue some ResourceWarnings which would otherwise get @@ -475,7 +491,7 @@ def test_func(): # failures. support.gc_collect() - remove_testfn(result.test_name, ns.verbose) + remove_testfn(result.test_name, runtests.verbose) if gc.garbage: support.environment_altered = True @@ -502,21 +518,22 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, pgo = runtests.pgo if pgo: display_failure = False + quiet = runtests.quiet test_name = result.test_name try: clear_caches() support.gc_collect() - with save_env(test_name, runtests, ns): + with save_env(test_name, runtests): _load_run_test(result, runtests, ns) except support.ResourceDenied as msg: - if not ns.quiet and not pgo: + if not quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as msg: - if not ns.quiet and not pgo: + if not quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.SKIPPED return diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index ecdde3aa523098..4bff7f99964119 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -127,7 +127,7 @@ def worker_process(worker_json: str) -> NoReturn: test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests - setup_test_dir(ns.testdir) + setup_test_dir(runtests.test_dir) setup_tests(runtests, ns) if runtests.rerun: @@ -136,7 +136,6 @@ def worker_process(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) else: print(f"Re-running {test_name} in verbose mode", flush=True) - ns.verbose = True result = run_single_test(test_name, runtests, ns) print() # Force a newline (just in case) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index f640362cb2df7f..59bbf2c81a167a 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -18,7 +18,7 @@ UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" -def setup_test_dir(testdir): +def setup_test_dir(testdir: str | None) -> None: if testdir: # Prepend test directory to sys.path, so runtest() will be able # to locate tests @@ -68,7 +68,7 @@ def setup_tests(runtests, ns): if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) - if ns.huntrleaks: + if runtests.hunt_refleak: unittest.BaseTestSuite._cleanup = False if ns.memlimit is not None: @@ -77,7 +77,7 @@ def setup_tests(runtests, ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) - support.suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) + support.suppress_msvcrt_asserts(runtests.verbose and runtests.verbose >= 2) support.use_resources = ns.use_resources @@ -102,7 +102,7 @@ def _test_audit_hook(name, args): support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) - if ns.xmlpath: + if runtests.junit_filename: from test.support.testresult import RegressionTestResult RegressionTestResult.USE_XML = True From 75cd86599bad05cb372aed9fccc3ff884cd38b70 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2023 10:21:42 -0500 Subject: [PATCH 585/598] Fix an ironic typo in a code comment. (gh-109186) --- Lib/random.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/random.py b/Lib/random.py index 586c3f7f9da938..84bbfc5df1bf23 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -827,7 +827,7 @@ def binomialvariate(self, n=1, p=0.5): return k # Acceptance-rejection test. - # Note, the original paper errorneously omits the call to log(v) + # Note, the original paper erroneously omits the call to log(v) # when comparing to the log of the rescaled binomial distribution. if not setup_complete: alpha = (2.83 + 5.1 / b) * spq From d3ed9921cdd8ac291fbfe3adf42f7730d3a14dbc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2023 17:50:04 -0500 Subject: [PATCH 586/598] Improve the sieve() recipe in the itertools docs (gh-109199) Lazier sieve --- Doc/library/itertools.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 42243715c2d93b..3cfc2602fe0694 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1030,13 +1030,16 @@ The following recipes have a more mathematical flavor: def sieve(n): "Primes less than n." # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + if n > 2: + yield 2 + start = 3 data = bytearray((0, 1)) * (n // 2) - data[:3] = 0, 0, 0 limit = math.isqrt(n) + 1 - for p in compress(range(limit), data): + for p in iter_index(data, 1, start, limit): + yield from iter_index(data, 1, start, p*p) data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - data[2] = 1 - return iter_index(data, 1) if n > 2 else iter([]) + start = p*p + yield from iter_index(data, 1, start) def factor(n): "Prime factors of n." From 24fa8f2046965b46c70a750a5a004708a63ac770 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 00:51:24 +0200 Subject: [PATCH 587/598] gh-109162: libregrtest: fix _decode_worker_job() (#109202) Decode also HuntRefleak() object inside the RunTests object. Add an unit test on huntrleaks with multiprocessing (-R -jN). --- Lib/test/libregrtest/runtest.py | 6 ++++++ Lib/test/libregrtest/runtest_mp.py | 2 +- Lib/test/test_regrtest.py | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 4f12176ced5c87..ab59d8123013da 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -249,6 +249,12 @@ def iter_tests(self): else: yield from self.tests + @staticmethod + def from_json_dict(json_dict): + if json_dict['hunt_refleak']: + json_dict['hunt_refleak'] = HuntRefleak(**json_dict['hunt_refleak']) + return RunTests(**json_dict) + # Minimum duration of a test to display its duration or to mention that # the test is running in background diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4bff7f99964119..4e3148fa8e2e67 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -68,7 +68,7 @@ def default(self, o: Any) -> dict[str, Any]: def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: if "__worker_job__" in d: d.pop('__worker_job__') - d['runtests'] = RunTests(**d['runtests']) + d['runtests'] = RunTests.from_json_dict(d['runtests']) return WorkerJob(**d) if "__namespace__" in d: d.pop('__namespace__') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8cced1f5185c2f..23896fdedaccac 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1018,12 +1018,16 @@ def test_run(self): stats=TestStats(4, 1), forever=True) - def check_leak(self, code, what): + def check_leak(self, code, what, *, multiprocessing=False): test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) - output = self.run_tests('--huntrleaks', '6:3:', test, + cmd = ['--huntrleaks', '6:3:'] + if multiprocessing: + cmd.append('-j1') + cmd.append(test) + output = self.run_tests(*cmd, exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) @@ -1039,7 +1043,7 @@ def check_leak(self, code, what): self.assertIn(line2, reflog) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') - def test_huntrleaks(self): + def check_huntrleaks(self, *, multiprocessing: bool): # test --huntrleaks code = textwrap.dedent(""" import unittest @@ -1050,7 +1054,13 @@ class RefLeakTest(unittest.TestCase): def test_leak(self): GLOBAL_LIST.append(object()) """) - self.check_leak(code, 'references') + self.check_leak(code, 'references', multiprocessing=multiprocessing) + + def test_huntrleaks(self): + self.check_huntrleaks(multiprocessing=False) + + def test_huntrleaks_mp(self): + self.check_huntrleaks(multiprocessing=True) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): From 0c0f254230bbc9532a7e78077a639a3ac940953c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 01:41:21 +0200 Subject: [PATCH 588/598] gh-109162: libregrtest: remove WorkerJob class (#109204) * Add attributes to Regrtest and RunTests: * gc_threshold * memory_limit * python_cmd * use_resources * Remove WorkerJob class. Add as_json() and from_json() methods to RunTests. A worker process now only uses RunTests for all parameters. * Add tests on support.set_memlimit() in test_support. Create _parse_memlimit() and also adds tests on it. * Remove 'ns' parameter from runtest.py. --- Lib/test/libregrtest/cmdline.py | 2 ++ Lib/test/libregrtest/main.py | 21 +++++++---- Lib/test/libregrtest/runtest.py | 56 ++++++++++++++++++++--------- Lib/test/libregrtest/runtest_mp.py | 58 ++++++------------------------ Lib/test/libregrtest/setup.py | 12 +++---- Lib/test/support/__init__.py | 24 +++++++------ Lib/test/test_support.py | 41 +++++++++++++++++++-- 7 files changed, 126 insertions(+), 88 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 71e7396d71d4a5..9afb13224975ee 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -177,6 +177,8 @@ def __init__(self, **kwargs) -> None: self.worker_json = None self.start = None self.timeout = None + self.memlimit = None + self.threshold = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index bb02101d97a2d2..4066d06c98600e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -100,6 +100,10 @@ def __init__(self, ns: Namespace): self.hunt_refleak = None self.test_dir: str | None = ns.testdir self.junit_filename: str | None = ns.xmlpath + self.memory_limit: str | None = ns.memlimit + self.gc_threshold: int | None = ns.threshold + self.use_resources: list[str] = ns.use_resources + self.python_cmd: list[str] | None = ns.python # tests self.tests = [] @@ -363,7 +367,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): - if self.ns.python: + if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( "Re-running failed tests is not supported with --python " @@ -453,12 +457,12 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = run_single_test(test_name, runtests, self.ns)') + cmd = ('result = run_single_test(test_name, runtests)') ns = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: - result = run_single_test(test_name, runtests, self.ns) + result = run_single_test(test_name, runtests) self.accumulate_result(result) @@ -876,9 +880,14 @@ def action_run_tests(self): quiet=self.quiet, hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, - junit_filename=self.junit_filename) - - setup_tests(runtests, self.ns) + junit_filename=self.junit_filename, + memory_limit=self.memory_limit, + gc_threshold=self.gc_threshold, + use_resources=self.use_resources, + python_cmd=self.python_cmd, + ) + + setup_tests(runtests) tracer = self.run_tests(runtests) self.display_result(runtests) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ab59d8123013da..dc574eda8b99f5 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -4,17 +4,18 @@ import gc import importlib import io +import json import os import sys import time import traceback import unittest +from typing import Any from test import support from test.support import TestStats from test.support import os_helper from test.support import threading_helper -from test.libregrtest.cmdline import Namespace from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import clear_caches, format_duration, print_warning @@ -230,6 +231,10 @@ class RunTests: hunt_refleak: HuntRefleak | None = None test_dir: str | None = None junit_filename: str | None = None + memory_limit: str | None = None + gc_threshold: int | None = None + use_resources: list[str] = None + python_cmd: list[str] | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -249,11 +254,32 @@ def iter_tests(self): else: yield from self.tests + def as_json(self): + return json.dumps(self, cls=_EncodeRunTests) + @staticmethod - def from_json_dict(json_dict): - if json_dict['hunt_refleak']: - json_dict['hunt_refleak'] = HuntRefleak(**json_dict['hunt_refleak']) - return RunTests(**json_dict) + def from_json(worker_json): + return json.loads(worker_json, object_hook=_decode_runtests) + + +class _EncodeRunTests(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, RunTests): + result = dataclasses.asdict(o) + result["__runtests__"] = True + return result + else: + return super().default(o) + + +def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: + if "__runtests__" in data: + data.pop('__runtests__') + if data['hunt_refleak']: + data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) + return RunTests(**data) + else: + return data # Minimum duration of a test to display its duration or to mention that @@ -320,7 +346,7 @@ def abs_module_name(test_name: str, test_dir: str | None) -> str: return 'test.' + test_name -def setup_support(runtests: RunTests, ns: Namespace): +def setup_support(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended support.set_match_tests(runtests.match_tests, runtests.ignore_tests) @@ -332,7 +358,7 @@ def setup_support(runtests: RunTests, ns: Namespace): support.junit_xml_list = None -def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: +def _runtest(result: TestResult, runtests: RunTests) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. verbose = runtests.verbose @@ -346,7 +372,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: faulthandler.dump_traceback_later(timeout, exit=True) try: - setup_support(runtests, ns) + setup_support(runtests) if output_on_failure: support.verbose = True @@ -366,7 +392,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # warnings will be written to sys.stderr below. print_warning.orig_stderr = stream - _runtest_env_changed_exc(result, runtests, ns, display_failure=False) + _runtest_env_changed_exc(result, runtests, display_failure=False) # Ignore output if the test passed successfully if result.state != State.PASSED: output = stream.getvalue() @@ -381,7 +407,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, runtests, ns, + _runtest_env_changed_exc(result, runtests, display_failure=not verbose) xml_list = support.junit_xml_list @@ -395,10 +421,9 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: support.junit_xml_list = None -def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestResult: +def run_single_test(test_name: str, runtests: RunTests) -> TestResult: """Run a single test. - ns -- regrtest namespace of options test_name -- the name of the test Returns a TestResult. @@ -410,7 +435,7 @@ def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestRe result = TestResult(test_name) pgo = runtests.pgo try: - _runtest(result, runtests, ns) + _runtest(result, runtests) except: if not pgo: msg = traceback.format_exc() @@ -472,7 +497,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: FOUND_GARBAGE = [] -def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: +def _load_run_test(result: TestResult, runtests: RunTests) -> None: # Load the test function, run the test function. module_name = abs_module_name(result.test_name, runtests.test_dir) @@ -513,7 +538,6 @@ def test_func(): def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, - ns: Namespace, display_failure: bool = True) -> None: # Detect environment changes, handle exceptions. @@ -532,7 +556,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, support.gc_collect() with save_env(test_name, runtests): - _load_run_test(result, runtests, ns) + _load_run_test(result, runtests) except support.ResourceDenied as msg: if not quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4e3148fa8e2e67..e4a9301656436b 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -47,49 +47,16 @@ @dataclasses.dataclass(slots=True) class WorkerJob: runtests: RunTests - namespace: Namespace -class _EncodeWorkerJob(json.JSONEncoder): - def default(self, o: Any) -> dict[str, Any]: - match o: - case WorkerJob(): - result = dataclasses.asdict(o) - result["__worker_job__"] = True - return result - case Namespace(): - result = vars(o) - result["__namespace__"] = True - return result - case _: - return super().default(o) - - -def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]: - if "__worker_job__" in d: - d.pop('__worker_job__') - d['runtests'] = RunTests.from_json_dict(d['runtests']) - return WorkerJob(**d) - if "__namespace__" in d: - d.pop('__namespace__') - return Namespace(**d) - else: - return d - - -def _parse_worker_json(worker_json: str) -> tuple[Namespace, str]: - return json.loads(worker_json, object_hook=_decode_worker_job) - - -def create_worker_process(worker_job: WorkerJob, +def create_worker_process(runtests: RunTests, output_file: TextIO, tmp_dir: str | None = None) -> subprocess.Popen: - ns = worker_job.namespace - python = ns.python - worker_json = json.dumps(worker_job, cls=_EncodeWorkerJob) + python_cmd = runtests.python_cmd + worker_json = runtests.as_json() - if python is not None: - executable = python + if python_cmd is not None: + executable = python_cmd else: executable = [sys.executable] cmd = [*executable, *support.args_from_interpreter_flags(), @@ -121,14 +88,12 @@ def create_worker_process(worker_job: WorkerJob, def worker_process(worker_json: str) -> NoReturn: - worker_job = _parse_worker_json(worker_json) - runtests = worker_job.runtests - ns = worker_job.namespace + runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests setup_test_dir(runtests.test_dir) - setup_tests(runtests, ns) + setup_tests(runtests) if runtests.rerun: if match_tests: @@ -137,7 +102,7 @@ def worker_process(worker_json: str) -> NoReturn: else: print(f"Re-running {test_name} in verbose mode", flush=True) - result = run_single_test(test_name, runtests, ns) + result = run_single_test(test_name, runtests) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON @@ -330,9 +295,6 @@ def _runtest(self, test_name: str) -> MultiprocessResult: if match_tests: kwargs['match_tests'] = match_tests worker_runtests = self.runtests.copy(tests=tests, **kwargs) - worker_job = WorkerJob( - worker_runtests, - namespace=self.ns) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. @@ -347,12 +309,12 @@ def _runtest(self, test_name: str) -> MultiprocessResult: tmp_dir = tempfile.mkdtemp(prefix="test_python_") tmp_dir = os.path.abspath(tmp_dir) try: - retcode = self._run_process(worker_job, stdout_file, tmp_dir) + retcode = self._run_process(worker_runtests, stdout_file, tmp_dir) finally: tmp_files = os.listdir(tmp_dir) os_helper.rmtree(tmp_dir) else: - retcode = self._run_process(worker_job, stdout_file) + retcode = self._run_process(worker_runtests, stdout_file) tmp_files = () stdout_file.seek(0) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 59bbf2c81a167a..594498333fe792 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -25,7 +25,7 @@ def setup_test_dir(testdir: str | None) -> None: sys.path.insert(0, os.path.abspath(testdir)) -def setup_tests(runtests, ns): +def setup_tests(runtests): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -71,15 +71,15 @@ def setup_tests(runtests, ns): if runtests.hunt_refleak: unittest.BaseTestSuite._cleanup = False - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) + if runtests.memory_limit is not None: + support.set_memlimit(runtests.memory_limit) - if ns.threshold is not None: - gc.set_threshold(ns.threshold) + if runtests.gc_threshold is not None: + gc.set_threshold(runtests.gc_threshold) support.suppress_msvcrt_asserts(runtests.verbose and runtests.verbose >= 2) - support.use_resources = ns.use_resources + support.use_resources = runtests.use_resources if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f40b73b0a799f1..6fc85ff5d704e1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -878,27 +878,31 @@ def inner(*args, **kwds): MAX_Py_ssize_t = sys.maxsize -def set_memlimit(limit): - global max_memuse - global real_max_memuse +def _parse_memlimit(limit: str) -> int: sizes = { 'k': 1024, 'm': _1M, 'g': _1G, 't': 1024*_1G, } - m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit, + m = re.match(r'(\d+(?:\.\d+)?) (K|M|G|T)b?$', limit, re.IGNORECASE | re.VERBOSE) if m is None: - raise ValueError('Invalid memory limit %r' % (limit,)) - memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) - real_max_memuse = memlimit - if memlimit > MAX_Py_ssize_t: - memlimit = MAX_Py_ssize_t + raise ValueError(f'Invalid memory limit: {limit!r}') + return int(float(m.group(1)) * sizes[m.group(2).lower()]) + +def set_memlimit(limit: str) -> None: + global max_memuse + global real_max_memuse + memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit %r too low to be useful' % (limit,)) + raise ValueError('Memory limit {limit!r} too low to be useful') + + real_max_memuse = memlimit + memlimit = min(memlimit, MAX_Py_ssize_t) max_memuse = memlimit + class _MemoryWatchdog: """An object which periodically watches the process' memory consumption and prints it out. diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 64280739f00946..5b57c5fd54a68d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -760,7 +760,45 @@ def recursive_function(depth): else: self.fail("RecursionError was not raised") - #self.assertEqual(available, 2) + def test_parse_memlimit(self): + parse = support._parse_memlimit + KiB = 1024 + MiB = KiB * 1024 + GiB = MiB * 1024 + TiB = GiB * 1024 + self.assertEqual(parse('0k'), 0) + self.assertEqual(parse('3k'), 3 * KiB) + self.assertEqual(parse('2.4m'), int(2.4 * MiB)) + self.assertEqual(parse('4g'), int(4 * GiB)) + self.assertEqual(parse('1t'), TiB) + + for limit in ('', '3', '3.5.10k', '10x'): + with self.subTest(limit=limit): + with self.assertRaises(ValueError): + parse(limit) + + def test_set_memlimit(self): + _4GiB = 4 * 1024 ** 3 + TiB = 1024 ** 4 + old_max_memuse = support.max_memuse + old_real_max_memuse = support.real_max_memuse + try: + if sys.maxsize > 2**32: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, _4GiB) + self.assertEqual(support.real_max_memuse, _4GiB) + + big = 2**100 // TiB + support.set_memlimit(f'{big}t') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, big * TiB) + else: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, _4GiB) + finally: + support.max_memuse = old_max_memuse + support.real_max_memuse = old_real_max_memuse # XXX -follows a list of untested API # make_legacy_pyc @@ -773,7 +811,6 @@ def recursive_function(depth): # EnvironmentVarGuard # transient_internet # run_with_locale - # set_memlimit # bigmemtest # precisionbigmemtest # bigaddrspacetest From 0553fdfe3040073307e8c53273041130148541d5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 02:24:38 +0200 Subject: [PATCH 589/598] gh-109162: Refactor libregrtest.runtest_mp (#109205) * Add attributes to Regrtest and RunTests: * fail_env_changed * num_workers * Rename MultiprocessTestRunner to RunWorkers. Add num_workers parameters to RunWorkers constructor. Remove RunWorkers.ns attribute. * Rename TestWorkerProcess to WorkerThread. * get_running() now returns a string like: "running (...): ...". * Regrtest.action_run_tests() now selects the number of worker processes, instead of the command line parser. --- Lib/test/libregrtest/cmdline.py | 6 +-- Lib/test/libregrtest/main.py | 44 +++++++++++++-------- Lib/test/libregrtest/runtest.py | 1 + Lib/test/libregrtest/runtest_mp.py | 62 ++++++++++++++---------------- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 9afb13224975ee..2835546fc713cf 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,5 @@ import argparse -import os +import os.path import shlex import sys from test.support import os_helper @@ -410,10 +410,6 @@ def _parse_args(args, **kwargs): if ns.timeout is not None: if ns.timeout <= 0: ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) if ns.use: for a in ns.use: for r in a: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 4066d06c98600e..1fa7b07a09d701 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -83,8 +83,18 @@ def __init__(self, ns: Namespace): self.fromfile: str | None = ns.fromfile self.starting_test: str | None = ns.start + # Run tests + if ns.use_mp is None: + num_workers = 0 # run sequentially + elif ns.use_mp <= 0: + num_workers = -1 # use the number of CPUs + else: + num_workers = ns.use_mp + self.num_workers: int = num_workers + # Options to run tests self.fail_fast: bool = ns.failfast + self.fail_env_changed: bool = ns.fail_env_changed self.forever: bool = ns.forever self.randomize: bool = ns.randomize self.random_seed: int | None = ns.random_seed @@ -150,7 +160,6 @@ def get_executed(self): | set(self.run_no_tests)) def accumulate_result(self, result, rerun=False): - fail_env_changed = self.ns.fail_env_changed test_name = result.test_name match result.state: @@ -167,7 +176,7 @@ def accumulate_result(self, result, rerun=False): case State.DID_NOT_RUN: self.run_no_tests.append(test_name) case _: - if result.is_failed(fail_env_changed): + if result.is_failed(self.fail_env_changed): self.bad.append(test_name) self.need_rerun.append(result) else: @@ -339,9 +348,8 @@ def get_rerun_match(self, rerun_list) -> FilterDict: def _rerun_failed_tests(self, need_rerun, runtests: RunTests): # Configure the runner to re-run tests - ns = self.ns - if ns.use_mp is None: - ns.use_mp = 1 + if self.num_workers == 0: + self.num_workers = 1 # Get tests to re-run tests = [result.test_name for result in need_rerun] @@ -363,7 +371,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.set_tests(runtests) - self._run_tests_mp(runtests) + self._run_tests_mp(runtests, self.num_workers) return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): @@ -471,7 +479,6 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): def run_tests_sequentially(self, runtests): ns = self.ns coverage = ns.trace - fail_env_changed = ns.fail_env_changed if coverage: import trace @@ -503,7 +510,7 @@ def run_tests_sequentially(self, runtests): if module not in save_modules and module.startswith("test."): support.unload(module) - if result.must_stop(self.fail_fast, fail_env_changed): + if result.must_stop(self.fail_fast, self.fail_env_changed): break previous_test = str(result) @@ -564,12 +571,10 @@ def no_tests_run(self): self.environment_changed)) def get_tests_state(self): - fail_env_changed = self.ns.fail_env_changed - result = [] if self.bad: result.append("FAILURE") - elif fail_env_changed and self.environment_changed: + elif self.fail_env_changed and self.environment_changed: result.append("ENV CHANGED") elif self.no_tests_run(): result.append("NO TESTS RAN") @@ -585,8 +590,9 @@ def get_tests_state(self): result = '%s then %s' % (self.first_state, result) return result - def _run_tests_mp(self, runtests: RunTests) -> None: - from test.libregrtest.runtest_mp import run_tests_multiprocess + def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: + from test.libregrtest.runtest_mp import RunWorkers + # If we're on windows and this is the parent runner (not a worker), # track the load average. if sys.platform == 'win32': @@ -600,7 +606,7 @@ def _run_tests_mp(self, runtests: RunTests) -> None: print(f'Failed to create WindowsLoadTracker: {error}') try: - run_tests_multiprocess(self, runtests) + RunWorkers(self, runtests, num_workers).run() finally: if self.win_load_tracker is not None: self.win_load_tracker.close() @@ -618,8 +624,8 @@ def set_tests(self, runtests: RunTests): def run_tests(self, runtests: RunTests): self.first_runtests = runtests self.set_tests(runtests) - if self.ns.use_mp: - self._run_tests_mp(runtests) + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) tracer = None else: tracer = self.run_tests_sequentially(runtests) @@ -843,7 +849,7 @@ def get_exitcode(self): exitcode = EXITCODE_BAD_TEST elif self.interrupted: exitcode = EXITCODE_INTERRUPTED - elif self.ns.fail_env_changed and self.environment_changed: + elif self.fail_env_changed and self.environment_changed: exitcode = EXITCODE_ENV_CHANGED elif self.no_tests_run(): exitcode = EXITCODE_NO_TESTS_RAN @@ -866,6 +872,10 @@ def action_run_tests(self): if self.randomize: print("Using random seed", self.random_seed) + if self.num_workers < 0: + # Use all cores + extras for tests that like to sleep + self.num_workers = 2 + (os.cpu_count() or 1) + runtests = RunTests( tuple(self.selected), fail_fast=self.fail_fast, diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dc574eda8b99f5..667701778d9a79 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -217,6 +217,7 @@ def get_rerun_match_tests(self) -> FilterTuple | None: class RunTests: tests: TestTuple fail_fast: bool = False + fail_env_changed: bool = False match_tests: FilterTuple | None = None ignore_tests: FilterTuple | None = None match_tests_dict: FilterDict | None = None diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index e4a9301656436b..28c05b5976e159 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -16,7 +16,6 @@ from test.support import os_helper from test.support import TestStats -from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( run_single_test, TestResult, State, PROGRESS_MIN_TIME, @@ -150,14 +149,13 @@ class ExitThread(Exception): pass -class TestWorkerProcess(threading.Thread): - def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: +class WorkerThread(threading.Thread): + def __init__(self, worker_id: int, runner: "RunWorkers") -> None: super().__init__() self.worker_id = worker_id self.runtests = runner.runtests self.pending = runner.pending self.output = runner.output - self.ns = runner.ns self.timeout = runner.worker_timeout self.regrtest = runner.regrtest self.current_test_name = None @@ -167,7 +165,7 @@ def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: self._stopped = False def __repr__(self) -> str: - info = [f'TestWorkerProcess #{self.worker_id}'] + info = [f'WorkerThread #{self.worker_id}'] if self.is_alive(): info.append("running") else: @@ -203,7 +201,7 @@ def _kill(self) -> None: else: popen.kill() except ProcessLookupError: - # popen.kill(): the process completed, the TestWorkerProcess thread + # popen.kill(): the process completed, the WorkerThread thread # read its exit status, but Popen.send_signal() read the returncode # just before Popen.wait() set returncode. pass @@ -362,7 +360,7 @@ def _runtest(self, test_name: str) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast - fail_env_changed = self.ns.fail_env_changed + fail_env_changed = self.runtests.fail_env_changed while not self._stopped: try: try: @@ -394,10 +392,10 @@ def _wait_completed(self) -> None: f"{exc!r}") def wait_stopped(self, start_time: float) -> None: - # bpo-38207: MultiprocessTestRunner.stop_workers() called self.stop() + # bpo-38207: RunWorkers.stop_workers() called self.stop() # which killed the process. Sometimes, killing the process from the # main thread does not interrupt popen.communicate() in - # TestWorkerProcess thread. This loop with a timeout is a workaround + # WorkerThread thread. This loop with a timeout is a workaround # for that. # # Moreover, if this method fails to join the thread, it is likely @@ -417,7 +415,7 @@ def wait_stopped(self, start_time: float) -> None: break -def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: +def get_running(workers: list[WorkerThread]) -> list[str]: running = [] for worker in workers: current_test_name = worker.current_test_name @@ -427,18 +425,17 @@ def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: if dt >= PROGRESS_MIN_TIME: text = '%s (%s)' % (current_test_name, format_duration(dt)) running.append(text) - return running + if not running: + return None + return f"running ({len(running)}): {', '.join(running)}" -class MultiprocessTestRunner: - def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: - ns = regrtest.ns - +class RunWorkers: + def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: self.regrtest = regrtest + self.log = regrtest.log + self.num_workers = num_workers self.runtests = runtests - self.rerun = runtests.rerun - self.log = self.regrtest.log - self.ns = ns self.output: queue.Queue[QueueOutput] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) @@ -453,9 +450,8 @@ def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: self.workers = None def start_workers(self) -> None: - use_mp = self.ns.use_mp - self.workers = [TestWorkerProcess(index, self) - for index in range(1, use_mp + 1)] + self.workers = [WorkerThread(index, self) + for index in range(1, self.num_workers + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" if self.timeout: msg += (" (timeout: %s, worker timeout: %s)" @@ -489,10 +485,11 @@ def _get_result(self) -> QueueOutput | None: except queue.Empty: pass - # display progress - running = get_running(self.workers) - if running and not pgo: - self.log('running: %s' % ', '.join(running)) + if not pgo: + # display progress + running = get_running(self.workers) + if running: + self.log(running) # all worker threads are done: consume pending results try: @@ -510,9 +507,10 @@ def display_result(self, mp_result: MultiprocessResult) -> None: text += ' (%s)' % mp_result.err_msg elif (result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) - running = get_running(self.workers) - if running and not pgo: - text += ' -- running: %s' % ', '.join(running) + if not pgo: + running = get_running(self.workers) + if running: + text += f' -- {running}' self.regrtest.display_progress(self.test_index, text) def _process_result(self, item: QueueOutput) -> bool: @@ -537,9 +535,9 @@ def _process_result(self, item: QueueOutput) -> bool: return result - def run_tests(self) -> None: + def run(self) -> None: fail_fast = self.runtests.fail_fast - fail_env_changed = self.ns.fail_env_changed + fail_env_changed = self.runtests.fail_env_changed self.start_workers() @@ -566,10 +564,6 @@ def run_tests(self) -> None: self.stop_workers() -def run_tests_multiprocess(regrtest: Regrtest, runtests: RunTests) -> None: - MultiprocessTestRunner(regrtest, runtests).run_tests() - - class EncodeTestResult(json.JSONEncoder): """Encode a TestResult (sub)class object into a JSON dict.""" From a341750078e6de5206560adcf53337e0172ae1c6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 03:07:05 +0200 Subject: [PATCH 590/598] gh-109162: Refactor libregrtest.Regrtest (#109206) * Add type hint types: TestName, StrPath, StrJSON. * Add attributes to Regrtest: * cmdline_args * coverage * coverage_dir * fail_rerun * next_single_filename * print_slowest * tmp_dir * want_rerun * want_run_leaks * Remove Regrtest.ns attribute. * Rename Regrtest methods: * cleanup() => cleanup_temp_dir() * create_temp_dir() => make_temp_dir() * set_temp_dir() => select_temp_dir() * Convert Regrtest methods to static methods: * cleanup_temp_dir() * display_header() * fix_umask() * get_rerun_match_tests() * list_tests() * make_temp_dir() * select_temp_dir() * Remove display_sanitizers() method: move code into display_header(). * Rename 'test_cwd' variable to 'work_dir'. --- Lib/test/libregrtest/cmdline.py | 2 + Lib/test/libregrtest/main.py | 174 ++++++++++++++--------------- Lib/test/libregrtest/pgo.py | 6 +- Lib/test/libregrtest/runtest.py | 43 +++---- Lib/test/libregrtest/runtest_mp.py | 10 +- 5 files changed, 120 insertions(+), 115 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2835546fc713cf..41d969625d04d2 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -179,6 +179,8 @@ def __init__(self, **kwargs) -> None: self.timeout = None self.memlimit = None self.threshold = None + self.fail_rerun = False + self.tempdir = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1fa7b07a09d701..e2e732ebd69e1c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -13,7 +13,7 @@ from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak, - FilterTuple, FilterDict, TestList) + FilterTuple, FilterDict, TestList, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, @@ -60,15 +60,14 @@ class Regrtest: on the command line. """ def __init__(self, ns: Namespace): - # Namespace of command line options - self.ns: Namespace = ns - # Actions self.want_header: bool = ns.header self.want_list_tests: bool = ns.list_tests self.want_list_cases: bool = ns.list_cases self.want_wait: bool = ns.wait self.want_cleanup: bool = ns.cleanup + self.want_rerun: bool = ns.rerun + self.want_run_leaks: bool = ns.runleaks # Select tests if ns.match_tests: @@ -80,10 +79,11 @@ def __init__(self, ns: Namespace): else: self.ignore_tests = None self.exclude: bool = ns.exclude - self.fromfile: str | None = ns.fromfile - self.starting_test: str | None = ns.start + self.fromfile: StrPath | None = ns.fromfile + self.starting_test: TestName | None = ns.start + self.cmdline_args: TestList = ns.args - # Run tests + # Workers if ns.use_mp is None: num_workers = 0 # run sequentially elif ns.use_mp <= 0: @@ -91,10 +91,12 @@ def __init__(self, ns: Namespace): else: num_workers = ns.use_mp self.num_workers: int = num_workers + self.worker_json: StrJSON | None = ns.worker_json # Options to run tests self.fail_fast: bool = ns.failfast self.fail_env_changed: bool = ns.fail_env_changed + self.fail_rerun: bool = ns.fail_rerun self.forever: bool = ns.forever self.randomize: bool = ns.randomize self.random_seed: int | None = ns.random_seed @@ -108,12 +110,15 @@ def __init__(self, ns: Namespace): self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks) else: self.hunt_refleak = None - self.test_dir: str | None = ns.testdir - self.junit_filename: str | None = ns.xmlpath + self.test_dir: StrPath | None = ns.testdir + self.junit_filename: StrPath | None = ns.xmlpath self.memory_limit: str | None = ns.memlimit self.gc_threshold: int | None = ns.threshold self.use_resources: list[str] = ns.use_resources self.python_cmd: list[str] | None = ns.python + self.coverage: bool = ns.trace + self.coverage_dir: StrPath | None = ns.coverdir + self.tmp_dir: StrPath | None = ns.tempdir # tests self.tests = [] @@ -135,8 +140,9 @@ def __init__(self, ns: Namespace): self.interrupted = False self.total_stats = TestStats() - # used by --slow - self.test_times = [] + # used by --slowest + self.test_times: list[tuple[float, TestName]] = [] + self.print_slowest: bool = ns.print_slow # used to display the progress bar "[ 3/100]" self.start_time = time.perf_counter() @@ -144,15 +150,15 @@ def __init__(self, ns: Namespace): self.test_count_width = 1 # used by --single - self.next_single_test = None - self.next_single_filename = None + self.single_test_run: bool = ns.single + self.next_single_test: TestName | None = None + self.next_single_filename: StrPath | None = None # used by --junit-xml self.testsuite_xml = None # misc self.win_load_tracker = None - self.tmp_dir = None def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) @@ -232,10 +238,7 @@ def display_progress(self, test_index, text): self.log(f"[{line}] {text}") def find_tests(self): - ns = self.ns - single = ns.single - - if single: + if self.single_test_run: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: with open(self.next_single_filename, 'r') as fp: @@ -261,19 +264,19 @@ def find_tests(self): if self.pgo: # add default PGO tests if no tests are specified - setup_pgo_tests(ns) + setup_pgo_tests(self.cmdline_args, self.pgo_extended) exclude_tests = set() if self.exclude: - for arg in ns.args: + for arg in self.cmdline_args: exclude_tests.add(arg) - ns.args = [] + self.cmdline_args = [] alltests = findtests(testdir=self.test_dir, exclude=exclude_tests) if not self.fromfile: - self.selected = self.tests or ns.args + self.selected = self.tests or self.cmdline_args if self.selected: self.selected = split_test_packages(self.selected) else: @@ -281,7 +284,7 @@ def find_tests(self): else: self.selected = self.tests - if single: + if self.single_test_run: self.selected = self.selected[:1] try: pos = alltests.index(self.selected[0]) @@ -303,8 +306,9 @@ def find_tests(self): random.seed(self.random_seed) random.shuffle(self.selected) - def list_tests(self): - for name in self.selected: + @staticmethod + def list_tests(tests: TestList): + for name in tests: print(name) def _list_cases(self, suite): @@ -337,7 +341,8 @@ def list_cases(self): print(count(len(skipped), "test"), "skipped:", file=stderr) printlist(skipped, file=stderr) - def get_rerun_match(self, rerun_list) -> FilterDict: + @staticmethod + def get_rerun_match(rerun_list) -> FilterDict: rerun_match_tests = {} for result in rerun_list: match_tests = result.get_rerun_match_tests() @@ -396,7 +401,6 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): def display_result(self, runtests): pgo = runtests.pgo - print_slow = self.ns.print_slow # If running the test suite for PGO then no one cares about results. if pgo: @@ -423,7 +427,7 @@ def display_result(self, runtests): print("All", end=' ') print(count(len(self.good), "test"), "OK.") - if print_slow: + if self.print_slowest: self.test_times.sort(reverse=True) print() print("10 slowest tests:") @@ -461,14 +465,14 @@ def display_result(self, runtests): print(count(len(self.run_no_tests), "test"), "run no tests:") printlist(self.run_no_tests) - def run_test(self, test_name: str, runtests: RunTests, tracer): + def run_test(self, test_name: TestName, runtests: RunTests, tracer): if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = ('result = run_single_test(test_name, runtests)') - ns = dict(locals()) - tracer.runctx(cmd, globals=globals(), locals=ns) - result = ns['result'] + namespace = dict(locals()) + tracer.runctx(cmd, globals=globals(), locals=namespace) + result = namespace['result'] else: result = run_single_test(test_name, runtests) @@ -477,10 +481,7 @@ def run_test(self, test_name: str, runtests: RunTests, tracer): return result def run_tests_sequentially(self, runtests): - ns = self.ns - coverage = ns.trace - - if coverage: + if self.coverage: import trace tracer = trace.Trace(trace=False, count=True) else: @@ -526,7 +527,8 @@ def run_tests_sequentially(self, runtests): return tracer - def display_header(self): + @staticmethod + def display_header(): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), @@ -538,9 +540,7 @@ def display_header(self): print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) - self.display_sanitizers() - def display_sanitizers(self): # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) @@ -642,9 +642,9 @@ def finalize_tests(self, tracer): if tracer is not None: results = tracer.results() results.write_results(show_missing=True, summary=True, - coverdir=self.ns.coverdir) + coverdir=self.coverage_dir) - if self.ns.runleaks: + if self.want_run_leaks: os.system("leaks %d" % os.getpid()) self.save_xml_result() @@ -722,7 +722,8 @@ def save_xml_result(self): for s in ET.tostringlist(root): f.write(s) - def fix_umask(self): + @staticmethod + def fix_umask(): if support.is_emscripten: # Emscripten has default umask 0o777, which breaks some tests. # see https://github.com/emscripten-core/emscripten/issues/17269 @@ -732,37 +733,34 @@ def fix_umask(self): else: os.umask(old_mask) - def set_temp_dir(self): - ns = self.ns - if ns.tempdir: - ns.tempdir = os.path.expanduser(ns.tempdir) - - if ns.tempdir: - self.tmp_dir = ns.tempdir - - if not self.tmp_dir: + @staticmethod + def select_temp_dir(tmp_dir): + if tmp_dir: + tmp_dir = os.path.expanduser(tmp_dir) + else: # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. if sysconfig.is_python_build(): - self.tmp_dir = sysconfig.get_config_var('abs_builddir') - if self.tmp_dir is None: + tmp_dir = sysconfig.get_config_var('abs_builddir') + if tmp_dir is None: # bpo-30284: On Windows, only srcdir is available. Using # abs_builddir mostly matters on UNIX when building Python # out of the source tree, especially when the source tree # is read only. - self.tmp_dir = sysconfig.get_config_var('srcdir') - self.tmp_dir = os.path.join(self.tmp_dir, 'build') + tmp_dir = sysconfig.get_config_var('srcdir') + tmp_dir = os.path.join(tmp_dir, 'build') else: - self.tmp_dir = tempfile.gettempdir() + tmp_dir = tempfile.gettempdir() - self.tmp_dir = os.path.abspath(self.tmp_dir) + return os.path.abspath(tmp_dir) def is_worker(self): - return (self.ns.worker_json is not None) + return (self.worker_json is not None) - def create_temp_dir(self): - os.makedirs(self.tmp_dir, exist_ok=True) + @staticmethod + def make_temp_dir(tmp_dir: StrPath, is_worker: bool): + os.makedirs(tmp_dir, exist_ok=True) # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel @@ -774,19 +772,20 @@ def create_temp_dir(self): else: nounce = os.getpid() - if self.is_worker(): - test_cwd = 'test_python_worker_{}'.format(nounce) + if is_worker: + work_dir = 'test_python_worker_{}'.format(nounce) else: - test_cwd = 'test_python_{}'.format(nounce) - test_cwd += os_helper.FS_NONASCII - test_cwd = os.path.join(self.tmp_dir, test_cwd) - return test_cwd + work_dir = 'test_python_{}'.format(nounce) + work_dir += os_helper.FS_NONASCII + work_dir = os.path.join(tmp_dir, work_dir) + return work_dir - def cleanup(self): + @staticmethod + def cleanup_temp_dir(tmp_dir: StrPath): import glob - path = os.path.join(glob.escape(self.tmp_dir), 'test_python_*') - print("Cleanup %s directory" % self.tmp_dir) + path = os.path.join(glob.escape(tmp_dir), 'test_python_*') + print("Cleanup %s directory" % tmp_dir) for name in glob.glob(path): if os.path.isdir(name): print("Remove directory: %s" % name) @@ -796,34 +795,33 @@ def cleanup(self): os_helper.unlink(name) def main(self, tests: TestList | None = None): - ns = self.ns self.tests = tests if self.junit_filename: support.junit_xml_list = self.testsuite_xml = [] - strip_py_suffix(ns.args) + strip_py_suffix(self.cmdline_args) - self.set_temp_dir() + self.tmp_dir = self.select_temp_dir(self.tmp_dir) self.fix_umask() if self.want_cleanup: - self.cleanup() + self.cleanup_temp_dir(self.tmp_dir) sys.exit(0) - test_cwd = self.create_temp_dir() + work_dir = self.make_temp_dir(self.tmp_dir, self.is_worker()) try: - # Run the tests in a context manager that temporarily changes the CWD - # to a temporary and writable directory. If it's not possible to - # create or change the CWD, the original CWD will be used. + # Run the tests in a context manager that temporarily changes the + # CWD to a temporary and writable directory. If it's not possible + # to create or change the CWD, the original CWD will be used. # The original CWD is available from os_helper.SAVEDCWD. - with os_helper.temp_cwd(test_cwd, quiet=True): - # When using multiprocessing, worker processes will use test_cwd - # as their parent temporary directory. So when the main process - # exit, it removes also subdirectories of worker processes. - ns.tempdir = test_cwd + with os_helper.temp_cwd(work_dir, quiet=True): + # When using multiprocessing, worker processes will use + # work_dir as their parent temporary directory. So when the + # main process exit, it removes also subdirectories of worker + # processes. self._main() except SystemExit as exc: @@ -853,7 +851,7 @@ def get_exitcode(self): exitcode = EXITCODE_ENV_CHANGED elif self.no_tests_run(): exitcode = EXITCODE_NO_TESTS_RAN - elif self.rerun and self.ns.fail_rerun: + elif self.rerun and self.fail_rerun: exitcode = EXITCODE_RERUN_FAIL return exitcode @@ -865,8 +863,8 @@ def action_run_tests(self): # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.pgo or self.quiet or self.ns.single - or self.tests or self.ns.args)): + or not(self.pgo or self.quiet or self.single_test_run + or self.tests or self.cmdline_args)): self.display_header() if self.randomize: @@ -903,7 +901,7 @@ def action_run_tests(self): self.display_result(runtests) need_rerun = self.need_rerun - if self.ns.rerun and need_rerun: + if self.want_rerun and need_rerun: self.rerun_failed_tests(need_rerun, runtests) self.display_summary() @@ -912,7 +910,7 @@ def action_run_tests(self): def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import worker_process - worker_process(self.ns.worker_json) + worker_process(self.worker_json) return if self.want_wait: @@ -923,7 +921,7 @@ def _main(self): exitcode = 0 if self.want_list_tests: - self.list_tests() + self.list_tests(self.selected) elif self.want_list_cases: self.list_cases() else: diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py index 42ce5fba7a97c3..cabbba73d5eff5 100644 --- a/Lib/test/libregrtest/pgo.py +++ b/Lib/test/libregrtest/pgo.py @@ -50,7 +50,7 @@ 'test_xml_etree_c', ] -def setup_pgo_tests(ns): - if not ns.args and not ns.pgo_extended: +def setup_pgo_tests(cmdline_args, pgo_extended: bool): + if not cmdline_args and not pgo_extended: # run default set of tests for PGO training - ns.args = PGO_TESTS[:] + cmdline_args[:] = PGO_TESTS[:] diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 667701778d9a79..6607e912330b52 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -20,20 +20,23 @@ from test.libregrtest.utils import clear_caches, format_duration, print_warning -TestTuple = list[str] -TestList = list[str] +StrJSON = str +StrPath = str +TestName = str +TestTuple = tuple[TestName, ...] +TestList = list[TestName] # --match and --ignore options: list of patterns # ('*' joker character can be used) -FilterTuple = tuple[str, ...] -FilterDict = dict[str, FilterTuple] +FilterTuple = tuple[TestName, ...] +FilterDict = dict[TestName, FilterTuple] @dataclasses.dataclass(slots=True, frozen=True) class HuntRefleak: warmups: int runs: int - filename: str + filename: StrPath # Avoid enum.Enum to reduce the number of imports when tests are run @@ -110,7 +113,7 @@ def normalize_test_name(test_full_name, *, is_error=False): @dataclasses.dataclass(slots=True) class TestResult: - test_name: str + test_name: TestName state: str | None = None # Test duration in seconds duration: float | None = None @@ -230,8 +233,8 @@ class RunTests: verbose: bool = False quiet: bool = False hunt_refleak: HuntRefleak | None = None - test_dir: str | None = None - junit_filename: str | None = None + test_dir: StrPath | None = None + junit_filename: StrPath | None = None memory_limit: str | None = None gc_threshold: int | None = None use_resources: list[str] = None @@ -255,11 +258,11 @@ def iter_tests(self): else: yield from self.tests - def as_json(self): + def as_json(self) -> StrJSON: return json.dumps(self, cls=_EncodeRunTests) @staticmethod - def from_json(worker_json): + def from_json(worker_json: StrJSON) -> 'RunTests': return json.loads(worker_json, object_hook=_decode_runtests) @@ -292,7 +295,7 @@ def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: # Beware this can't generally be done for any directory with sub-tests as the # __init__.py may do things which alter what tests are to be run. -SPLITTESTDIRS = { +SPLITTESTDIRS: set[TestName] = { "test_asyncio", "test_concurrent_futures", "test_multiprocessing_fork", @@ -305,8 +308,9 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(*, testdir: str | None =None, exclude=(), - split_test_dirs=SPLITTESTDIRS, base_mod=""): +def findtests(*, testdir: StrPath | None = None, exclude=(), + split_test_dirs: set[TestName] = SPLITTESTDIRS, + base_mod: str = "") -> TestList: """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) tests = [] @@ -318,13 +322,14 @@ def findtests(*, testdir: str | None =None, exclude=(), subdir = os.path.join(testdir, mod) mod = f"{base_mod or 'test'}.{mod}" tests.extend(findtests(testdir=subdir, exclude=exclude, - split_test_dirs=split_test_dirs, base_mod=mod)) + split_test_dirs=split_test_dirs, + base_mod=mod)) elif ext in (".py", ""): tests.append(f"{base_mod}.{mod}" if base_mod else mod) return sorted(tests) -def split_test_packages(tests, *, testdir: str | None = None, exclude=(), +def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), split_test_dirs=SPLITTESTDIRS): testdir = findtestdir(testdir) splitted = [] @@ -339,7 +344,7 @@ def split_test_packages(tests, *, testdir: str | None = None, exclude=(), return splitted -def abs_module_name(test_name: str, test_dir: str | None) -> str: +def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: if test_name.startswith('test.') or test_dir: return test_name else: @@ -422,7 +427,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: support.junit_xml_list = None -def run_single_test(test_name: str, runtests: RunTests) -> TestResult: +def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: """Run a single test. test_name -- the name of the test @@ -457,7 +462,7 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(test_name: str, runtests: RunTests): +def save_env(test_name: TestName, runtests: RunTests): return saved_test_environment(test_name, runtests.verbose, runtests.quiet, pgo=runtests.pgo) @@ -608,7 +613,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, result.state = State.PASSED -def remove_testfn(test_name: str, verbose: int) -> None: +def remove_testfn(test_name: TestName, verbose: int) -> None: # Try to clean up os_helper.TESTFN if left behind. # # While tests shouldn't leave any files or directories behind, when a test diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 28c05b5976e159..2575f1811a4930 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -19,7 +19,7 @@ from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( run_single_test, TestResult, State, PROGRESS_MIN_TIME, - FilterTuple, RunTests) + FilterTuple, RunTests, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.utils import format_duration, print_warning @@ -50,7 +50,7 @@ class WorkerJob: def create_worker_process(runtests: RunTests, output_file: TextIO, - tmp_dir: str | None = None) -> subprocess.Popen: + tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -86,7 +86,7 @@ def create_worker_process(runtests: RunTests, return subprocess.Popen(cmd, **kw) -def worker_process(worker_json: str) -> NoReturn: +def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests @@ -222,7 +222,7 @@ def mp_result_error( return MultiprocessResult(test_result, stdout, err_msg) def _run_process(self, worker_job, output_file: TextIO, - tmp_dir: str | None = None) -> int: + tmp_dir: StrPath | None = None) -> int: try: popen = create_worker_process(worker_job, output_file, tmp_dir) @@ -273,7 +273,7 @@ def _run_process(self, worker_job, output_file: TextIO, self._popen = None self.current_test_name = None - def _runtest(self, test_name: str) -> MultiprocessResult: + def _runtest(self, test_name: TestName) -> MultiprocessResult: self.current_test_name = test_name if sys.platform == 'win32': From db5bfe91f822bb06b4c8fe7c58e649694c854bf2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 04:30:43 +0200 Subject: [PATCH 591/598] gh-109162: libregrtest: add TestResults class (#109208) * Add TestResults class. * Move Regrtest methods to TestResults: * accumulate_result(): now takes a RunTests parameter * get_executed() * no_tests_run() * Add methods to TestResults: * add_junit() * display_result() * display_summary() * need_rerun() * prepare_rerun() * write_junit() * Rename 'need_rerun' attribute to 'bad_results'. * Rename 'total_stats' attribute to 'stats'. --- Lib/test/libregrtest/main.py | 303 ++++------------------------- Lib/test/libregrtest/results.py | 259 ++++++++++++++++++++++++ Lib/test/libregrtest/runtest_mp.py | 18 +- 3 files changed, 306 insertions(+), 274 deletions(-) create mode 100644 Lib/test/libregrtest/results.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e2e732ebd69e1c..75c3d0e8350ade 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,14 +12,14 @@ from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, - PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak, - FilterTuple, FilterDict, TestList, StrPath, StrJSON, TestName) + PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, + FilterTuple, TestList, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests +from test.libregrtest.results import TestResults from test.libregrtest.utils import (strip_py_suffix, count, format_duration, printlist, get_build_info) from test import support -from test.support import TestStats from test.support import os_helper from test.support import threading_helper @@ -29,12 +29,6 @@ # Must be smaller than buildbot "1200 seconds without output" limit. EXIT_TIMEOUT = 120.0 -EXITCODE_BAD_TEST = 2 -EXITCODE_ENV_CHANGED = 3 -EXITCODE_NO_TESTS_RAN = 4 -EXITCODE_RERUN_FAIL = 5 -EXITCODE_INTERRUPTED = 130 - class Regrtest: """Execute a test suite. @@ -122,26 +116,15 @@ def __init__(self, ns: Namespace): # tests self.tests = [] - self.selected = [] + self.selected: TestList = [] self.first_runtests: RunTests | None = None # test results - self.good: TestList = [] - self.bad: TestList = [] - self.rerun_bad: TestList = [] - self.skipped: TestList = [] - self.resource_denied: TestList = [] - self.environment_changed: TestList = [] - self.run_no_tests: TestList = [] - self.rerun: TestList = [] - - self.need_rerun: list[TestResult] = [] + self.results: TestResults = TestResults() + self.first_state: str | None = None - self.interrupted = False - self.total_stats = TestStats() # used by --slowest - self.test_times: list[tuple[float, TestName]] = [] self.print_slowest: bool = ns.print_slow # used to display the progress bar "[ 3/100]" @@ -154,57 +137,9 @@ def __init__(self, ns: Namespace): self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - # used by --junit-xml - self.testsuite_xml = None - # misc self.win_load_tracker = None - def get_executed(self): - return (set(self.good) | set(self.bad) | set(self.skipped) - | set(self.resource_denied) | set(self.environment_changed) - | set(self.run_no_tests)) - - def accumulate_result(self, result, rerun=False): - test_name = result.test_name - - match result.state: - case State.PASSED: - self.good.append(test_name) - case State.ENV_CHANGED: - self.environment_changed.append(test_name) - case State.SKIPPED: - self.skipped.append(test_name) - case State.RESOURCE_DENIED: - self.resource_denied.append(test_name) - case State.INTERRUPTED: - self.interrupted = True - case State.DID_NOT_RUN: - self.run_no_tests.append(test_name) - case _: - if result.is_failed(self.fail_env_changed): - self.bad.append(test_name) - self.need_rerun.append(result) - else: - raise ValueError(f"invalid test state: {result.state!r}") - - if result.has_meaningful_duration() and not rerun: - self.test_times.append((result.duration, test_name)) - if result.stats is not None: - self.total_stats.accumulate(result.stats) - if rerun: - self.rerun.append(test_name) - - xml_data = result.xml_data - if xml_data: - import xml.etree.ElementTree as ET - for e in xml_data: - try: - self.testsuite_xml.append(ET.fromstring(e)) - except ET.ParseError: - print(xml_data, file=sys.__stderr__) - raise - def log(self, line=''): empty = not line @@ -232,7 +167,7 @@ def display_progress(self, test_index, text): # "[ 51/405/1] test_tcl passed" line = f"{test_index:{self.test_count_width}}{self.test_count_text}" - fails = len(self.bad) + len(self.environment_changed) + fails = len(self.results.bad) + len(self.results.env_changed) if fails and not self.pgo: line = f"{line}/{fails}" self.log(f"[{line}] {text}") @@ -341,34 +276,17 @@ def list_cases(self): print(count(len(skipped), "test"), "skipped:", file=stderr) printlist(skipped, file=stderr) - @staticmethod - def get_rerun_match(rerun_list) -> FilterDict: - rerun_match_tests = {} - for result in rerun_list: - match_tests = result.get_rerun_match_tests() - # ignore empty match list - if match_tests: - rerun_match_tests[result.test_name] = match_tests - return rerun_match_tests - - def _rerun_failed_tests(self, need_rerun, runtests: RunTests): + def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests if self.num_workers == 0: self.num_workers = 1 - # Get tests to re-run - tests = [result.test_name for result in need_rerun] - match_tests_dict = self.get_rerun_match(need_rerun) - - # Clear previously failed tests - self.rerun_bad.extend(self.bad) - self.bad.clear() - self.need_rerun.clear() + tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( - tests=tuple(tests), + tests=tests, rerun=True, verbose=True, forever=False, @@ -379,7 +297,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests): self._run_tests_mp(runtests, self.num_workers) return runtests - def rerun_failed_tests(self, need_rerun, runtests: RunTests): + def rerun_failed_tests(self, runtests: RunTests): if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -388,82 +306,27 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests): ) return - self.first_state = self.get_tests_state() + self.first_state = self.get_state() print() - rerun_runtests = self._rerun_failed_tests(need_rerun, runtests) + rerun_runtests = self._rerun_failed_tests(runtests) - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) + if self.results.bad: + print(count(len(self.results.bad), 'test'), "failed again:") + printlist(self.results.bad) self.display_result(rerun_runtests) def display_result(self, runtests): - pgo = runtests.pgo - # If running the test suite for PGO then no one cares about results. - if pgo: + if runtests.pgo: return + state = self.get_state() print() - print("== Tests result: %s ==" % self.get_tests_state()) - - if self.interrupted: - print("Test suite interrupted by signal SIGINT.") - - omitted = set(self.selected) - self.get_executed() - if omitted: - print() - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - - if self.good and not self.quiet: - print() - if (not self.bad - and not self.skipped - and not self.interrupted - and len(self.good) > 1): - print("All", end=' ') - print(count(len(self.good), "test"), "OK.") - - if self.print_slowest: - self.test_times.sort(reverse=True) - print() - print("10 slowest tests:") - for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) - - if self.bad: - print() - print(count(len(self.bad), "test"), "failed:") - printlist(self.bad) - - if self.environment_changed: - print() - print("{} altered the execution environment:".format( - count(len(self.environment_changed), "test"))) - printlist(self.environment_changed) - - if self.skipped and not self.quiet: - print() - print(count(len(self.skipped), "test"), "skipped:") - printlist(self.skipped) - - if self.resource_denied and not self.quiet: - print() - print(count(len(self.resource_denied), "test"), "skipped (resource denied):") - printlist(self.resource_denied) - - if self.rerun: - print() - print("%s:" % count(len(self.rerun), "re-run test")) - printlist(self.rerun) - - if self.run_no_tests: - print() - print(count(len(self.run_no_tests), "test"), "run no tests:") - printlist(self.run_no_tests) + print(f"== Tests result: {state} ==") + + self.results.display_result(self.selected, self.quiet, self.print_slowest) def run_test(self, test_name: TestName, runtests: RunTests, tracer): if tracer is not None: @@ -476,7 +339,7 @@ def run_test(self, test_name: TestName, runtests: RunTests, tracer): else: result = run_single_test(test_name, runtests) - self.accumulate_result(result) + self.results.accumulate_result(result, runtests) return result @@ -566,29 +429,11 @@ def display_header(): if sanitizer and options is not None: print(f"== {env_var}={options!r}") - def no_tests_run(self): - return not any((self.good, self.bad, self.skipped, self.interrupted, - self.environment_changed)) - - def get_tests_state(self): - result = [] - if self.bad: - result.append("FAILURE") - elif self.fail_env_changed and self.environment_changed: - result.append("ENV CHANGED") - elif self.no_tests_run(): - result.append("NO TESTS RAN") - - if self.interrupted: - result.append("INTERRUPTED") - - if not result: - result.append("SUCCESS") - - result = ', '.join(result) + def get_state(self): + state = self.results.get_state(self.fail_env_changed) if self.first_state: - result = '%s then %s' % (self.first_state, result) - return result + state = f'{self.first_state} then {state}' + return state def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from test.libregrtest.runtest_mp import RunWorkers @@ -647,7 +492,8 @@ def finalize_tests(self, tracer): if self.want_run_leaks: os.system("leaks %d" % os.getpid()) - self.save_xml_result() + if self.junit_filename: + self.results.write_junit(self.junit_filename) def display_summary(self): duration = time.perf_counter() - self.start_time @@ -657,70 +503,11 @@ def display_summary(self): print() print("Total duration: %s" % format_duration(duration)) - # Total tests - total = self.total_stats - text = f'run={total.tests_run:,}' - if filtered: - text = f"{text} (filtered)" - stats = [text] - if total.failures: - stats.append(f'failures={total.failures:,}') - if total.skipped: - stats.append(f'skipped={total.skipped:,}') - print(f"Total tests: {' '.join(stats)}") - - # Total test files - all_tests = [self.good, self.bad, self.rerun, - self.skipped, - self.environment_changed, self.run_no_tests] - run = sum(map(len, all_tests)) - text = f'run={run}' - if not self.first_runtests.forever: - ntest = len(self.first_runtests.tests) - text = f"{text}/{ntest}" - if filtered: - text = f"{text} (filtered)" - report = [text] - for name, tests in ( - ('failed', self.bad), - ('env_changed', self.environment_changed), - ('skipped', self.skipped), - ('resource_denied', self.resource_denied), - ('rerun', self.rerun), - ('run_no_tests', self.run_no_tests), - ): - if tests: - report.append(f'{name}={len(tests)}') - print(f"Total test files: {' '.join(report)}") + self.results.display_summary(self.first_runtests, filtered) # Result - result = self.get_tests_state() - print(f"Result: {result}") - - def save_xml_result(self): - if not self.junit_filename and not self.testsuite_xml: - return - - import xml.etree.ElementTree as ET - root = ET.Element("testsuites") - - # Manually count the totals for the overall summary - totals = {'tests': 0, 'errors': 0, 'failures': 0} - for suite in self.testsuite_xml: - root.append(suite) - for k in totals: - try: - totals[k] += int(suite.get(k, 0)) - except ValueError: - pass - - for k, v in totals.items(): - root.set(k, str(v)) - - xmlpath = os.path.join(os_helper.SAVEDCWD, self.junit_filename) - with open(xmlpath, 'wb') as f: - for s in ET.tostringlist(root): - f.write(s) + state = self.get_state() + print(f"Result: {state}") @staticmethod def fix_umask(): @@ -795,10 +582,10 @@ def cleanup_temp_dir(tmp_dir: StrPath): os_helper.unlink(name) def main(self, tests: TestList | None = None): - self.tests = tests + if self.junit_filename and not os.path.isabs(self.junit_filename): + self.junit_filename = os.path.abspath(self.junit_filename) - if self.junit_filename: - support.junit_xml_list = self.testsuite_xml = [] + self.tests = tests strip_py_suffix(self.cmdline_args) @@ -841,20 +628,6 @@ def getloadavg(self): return None - def get_exitcode(self): - exitcode = 0 - if self.bad: - exitcode = EXITCODE_BAD_TEST - elif self.interrupted: - exitcode = EXITCODE_INTERRUPTED - elif self.fail_env_changed and self.environment_changed: - exitcode = EXITCODE_ENV_CHANGED - elif self.no_tests_run(): - exitcode = EXITCODE_NO_TESTS_RAN - elif self.rerun and self.fail_rerun: - exitcode = EXITCODE_RERUN_FAIL - return exitcode - def action_run_tests(self): if self.hunt_refleak and self.hunt_refleak.warmups < 3: msg = ("WARNING: Running tests with --huntrleaks/-R and " @@ -900,9 +673,8 @@ def action_run_tests(self): tracer = self.run_tests(runtests) self.display_result(runtests) - need_rerun = self.need_rerun - if self.want_rerun and need_rerun: - self.rerun_failed_tests(need_rerun, runtests) + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) self.display_summary() self.finalize_tests(tracer) @@ -926,7 +698,8 @@ def _main(self): self.list_cases() else: self.action_run_tests() - exitcode = self.get_exitcode() + exitcode = self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) sys.exit(exitcode) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py new file mode 100644 index 00000000000000..1df15c23770cc1 --- /dev/null +++ b/Lib/test/libregrtest/results.py @@ -0,0 +1,259 @@ +import sys +from test.support import TestStats + +from test.libregrtest.runtest import ( + TestName, TestTuple, TestList, FilterDict, StrPath, State, + TestResult, RunTests) +from test.libregrtest.utils import printlist, count, format_duration + + +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 + + +class TestResults: + def __init__(self): + self.bad: TestList = [] + self.good: TestList = [] + self.rerun_bad: TestList = [] + self.skipped: TestList = [] + self.resource_denied: TestList = [] + self.env_changed: TestList = [] + self.run_no_tests: TestList = [] + self.rerun: TestList = [] + self.bad_results: list[TestResult] = [] + + self.interrupted: bool = False + self.test_times: list[tuple[float, TestName]] = [] + self.stats = TestStats() + # used by --junit-xml + self.testsuite_xml: list[str] = [] + + def get_executed(self): + return (set(self.good) | set(self.bad) | set(self.skipped) + | set(self.resource_denied) | set(self.env_changed) + | set(self.run_no_tests)) + + def no_tests_run(self): + return not any((self.good, self.bad, self.skipped, self.interrupted, + self.env_changed)) + + def get_state(self, fail_env_changed): + state = [] + if self.bad: + state.append("FAILURE") + elif fail_env_changed and self.env_changed: + state.append("ENV CHANGED") + elif self.no_tests_run(): + state.append("NO TESTS RAN") + + if self.interrupted: + state.append("INTERRUPTED") + if not state: + state.append("SUCCESS") + + return ', '.join(state) + + def get_exitcode(self, fail_env_changed, fail_rerun): + exitcode = 0 + if self.bad: + exitcode = EXITCODE_BAD_TEST + elif self.interrupted: + exitcode = EXITCODE_INTERRUPTED + elif fail_env_changed and self.env_changed: + exitcode = EXITCODE_ENV_CHANGED + elif self.no_tests_run(): + exitcode = EXITCODE_NO_TESTS_RAN + elif fail_rerun and self.rerun: + exitcode = EXITCODE_RERUN_FAIL + return exitcode + + def accumulate_result(self, result: TestResult, runtests: RunTests): + test_name = result.test_name + rerun = runtests.rerun + fail_env_changed = runtests.fail_env_changed + + match result.state: + case State.PASSED: + self.good.append(test_name) + case State.ENV_CHANGED: + self.env_changed.append(test_name) + case State.SKIPPED: + self.skipped.append(test_name) + case State.RESOURCE_DENIED: + self.resource_denied.append(test_name) + case State.INTERRUPTED: + self.interrupted = True + case State.DID_NOT_RUN: + self.run_no_tests.append(test_name) + case _: + if result.is_failed(fail_env_changed): + self.bad.append(test_name) + self.bad_results.append(result) + else: + raise ValueError(f"invalid test state: {result.state!r}") + + if result.has_meaningful_duration() and not rerun: + self.test_times.append((result.duration, test_name)) + if result.stats is not None: + self.stats.accumulate(result.stats) + if rerun: + self.rerun.append(test_name) + + xml_data = result.xml_data + if xml_data: + self.add_junit(result.xml_data) + + def need_rerun(self): + return bool(self.bad_results) + + def prepare_rerun(self) -> (TestTuple, FilterDict): + tests: TestList = [] + match_tests_dict = {} + for result in self.bad_results: + tests.append(result.test_name) + + match_tests = result.get_rerun_match_tests() + # ignore empty match list + if match_tests: + match_tests_dict[result.test_name] = match_tests + + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.bad_results.clear() + + return (tuple(tests), match_tests_dict) + + def add_junit(self, xml_data: list[str]): + import xml.etree.ElementTree as ET + for e in xml_data: + try: + self.testsuite_xml.append(ET.fromstring(e)) + except ET.ParseError: + print(xml_data, file=sys.__stderr__) + raise + + def write_junit(self, filename: StrPath): + if not self.testsuite_xml: + # Don't create empty XML file + return + + import xml.etree.ElementTree as ET + root = ET.Element("testsuites") + + # Manually count the totals for the overall summary + totals = {'tests': 0, 'errors': 0, 'failures': 0} + for suite in self.testsuite_xml: + root.append(suite) + for k in totals: + try: + totals[k] += int(suite.get(k, 0)) + except ValueError: + pass + + for k, v in totals.items(): + root.set(k, str(v)) + + with open(filename, 'wb') as f: + for s in ET.tostringlist(root): + f.write(s) + + def display_result(self, tests: TestList, quiet: bool, print_slowest: bool): + if self.interrupted: + print("Test suite interrupted by signal SIGINT.") + + omitted = set(tests) - self.get_executed() + if omitted: + print() + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + + if self.good and not quiet: + print() + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): + print("All", end=' ') + print(count(len(self.good), "test"), "OK.") + + if print_slowest: + self.test_times.sort(reverse=True) + print() + print("10 slowest tests:") + for test_time, test in self.test_times[:10]: + print("- %s: %s" % (test, format_duration(test_time))) + + if self.bad: + print() + print(count(len(self.bad), "test"), "failed:") + printlist(self.bad) + + if self.env_changed: + print() + print("{} altered the execution environment:".format( + count(len(self.env_changed), "test"))) + printlist(self.env_changed) + + if self.skipped and not quiet: + print() + print(count(len(self.skipped), "test"), "skipped:") + printlist(self.skipped) + + if self.resource_denied and not quiet: + print() + print(count(len(self.resource_denied), "test"), "skipped (resource denied):") + printlist(self.resource_denied) + + if self.rerun: + print() + print("%s:" % count(len(self.rerun), "re-run test")) + printlist(self.rerun) + + if self.run_no_tests: + print() + print(count(len(self.run_no_tests), "test"), "run no tests:") + printlist(self.run_no_tests) + + def display_summary(self, first_runtests: RunTests, filtered: bool): + # Total tests + stats = self.stats + text = f'run={stats.tests_run:,}' + if filtered: + text = f"{text} (filtered)" + report = [text] + if stats.failures: + report.append(f'failures={stats.failures:,}') + if stats.skipped: + report.append(f'skipped={stats.skipped:,}') + report = ' '.join(report) + print(f"Total tests: {report}") + + # Total test files + all_tests = [self.good, self.bad, self.rerun, + self.skipped, + self.env_changed, self.run_no_tests] + run = sum(map(len, all_tests)) + text = f'run={run}' + if not first_runtests.forever: + ntest = len(first_runtests.tests) + text = f"{text}/{ntest}" + if filtered: + text = f"{text} (filtered)" + report = [text] + for name, tests in ( + ('failed', self.bad), + ('env_changed', self.env_changed), + ('skipped', self.skipped), + ('resource_denied', self.resource_denied), + ('rerun', self.rerun), + ('run_no_tests', self.run_no_tests), + ): + if tests: + report.append(f'{name}={len(tests)}') + report = ' '.join(report) + print(f"Total test files: {report}") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 2575f1811a4930..179b6104a8f79a 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,6 +21,7 @@ run_single_test, TestResult, State, PROGRESS_MIN_TIME, FilterTuple, RunTests, StrPath, StrJSON, TestName) from test.libregrtest.setup import setup_tests, setup_test_dir +from test.libregrtest.results import TestResults from test.libregrtest.utils import format_duration, print_warning if sys.platform == 'win32': @@ -157,7 +158,7 @@ def __init__(self, worker_id: int, runner: "RunWorkers") -> None: self.pending = runner.pending self.output = runner.output self.timeout = runner.worker_timeout - self.regrtest = runner.regrtest + self.log = runner.log self.current_test_name = None self.start_time = None self._popen = None @@ -408,8 +409,7 @@ def wait_stopped(self, start_time: float) -> None: if not self.is_alive(): break dt = time.monotonic() - start_time - self.regrtest.log(f"Waiting for {self} thread " - f"for {format_duration(dt)}") + self.log(f"Waiting for {self} thread for {format_duration(dt)}") if dt > JOIN_TIMEOUT: print_warning(f"Failed to join {self} in {format_duration(dt)}") break @@ -432,8 +432,9 @@ def get_running(workers: list[WorkerThread]) -> list[str]: class RunWorkers: def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: - self.regrtest = regrtest + self.results: TestResults = regrtest.results self.log = regrtest.log + self.display_progress = regrtest.display_progress self.num_workers = num_workers self.runtests = runtests self.output: queue.Queue[QueueOutput] = queue.Queue() @@ -511,23 +512,22 @@ def display_result(self, mp_result: MultiprocessResult) -> None: running = get_running(self.workers) if running: text += f' -- {running}' - self.regrtest.display_progress(self.test_index, text) + self.display_progress(self.test_index, text) def _process_result(self, item: QueueOutput) -> bool: """Returns True if test runner must stop.""" - rerun = self.runtests.rerun if item[0]: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") result = TestResult("", state=State.MULTIPROCESSING_ERROR) - self.regrtest.accumulate_result(result, rerun=rerun) + self.results.accumulate_result(result, self.runtests) return result self.test_index += 1 mp_result = item[1] result = mp_result.result - self.regrtest.accumulate_result(result, rerun=rerun) + self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) if mp_result.worker_stdout: @@ -553,7 +553,7 @@ def run(self) -> None: break except KeyboardInterrupt: print() - self.regrtest.interrupted = True + self.results.interrupted = True finally: if self.timeout is not None: faulthandler.cancel_dump_traceback_later() From 0eab2427b149cd46e0dee3efbb6b2cfca2a4f723 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 05:04:26 +0200 Subject: [PATCH 592/598] gh-109162: libregrtest: add Logger class (#109212) * Add Logger class in a new logger.py file. * Move Regrtest attributes to Logger: * start_time * test_count_text * test_count_width * win_load_tracker * Move Regrtest method to Logger: * log() * getloadavg(): rename to get_load_avg() * set_tests() * Add methods to the Logger class: * start_load_tracker() * stop_load_tracker() --- Lib/test/libregrtest/logger.py | 69 ++++++++++++++ Lib/test/libregrtest/main.py | 148 ++++++++++------------------- Lib/test/libregrtest/runtest_mp.py | 2 +- Lib/test/libregrtest/utils.py | 3 + 4 files changed, 123 insertions(+), 99 deletions(-) create mode 100644 Lib/test/libregrtest/logger.py diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py new file mode 100644 index 00000000000000..c4498a43545545 --- /dev/null +++ b/Lib/test/libregrtest/logger.py @@ -0,0 +1,69 @@ +import os +import time + +from test.libregrtest.runtest import RunTests +from test.libregrtest.utils import print_warning, MS_WINDOWS + +if MS_WINDOWS: + from test.libregrtest.win_utils import WindowsLoadTracker + + +class Logger: + def __init__(self): + self.start_time = time.perf_counter() + self.test_count_text = '' + self.test_count_width = 3 + self.win_load_tracker = None + + def log(self, line: str = '') -> None: + empty = not line + + # add the system load prefix: "load avg: 1.80 " + load_avg = self.get_load_avg() + if load_avg is not None: + line = f"load avg: {load_avg:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " + test_time = time.perf_counter() - self.start_time + + mins, secs = divmod(int(test_time), 60) + hours, mins = divmod(mins, 60) + test_time = "%d:%02d:%02d" % (hours, mins, secs) + + line = f"{test_time} {line}" + if empty: + line = line[:-1] + + print(line, flush=True) + + def get_load_avg(self) -> float | None: + if hasattr(os, 'getloadavg'): + return os.getloadavg()[0] + if self.win_load_tracker is not None: + return self.win_load_tracker.getloadavg() + return None + + def set_tests(self, runtests: RunTests) -> None: + if runtests.forever: + self.test_count_text = '' + self.test_count_width = 3 + else: + self.test_count_text = '/{}'.format(len(runtests.tests)) + self.test_count_width = len(self.test_count_text) - 1 + + def start_load_tracker(self) -> None: + if not MS_WINDOWS: + return + + try: + self.win_load_tracker = WindowsLoadTracker() + except PermissionError as error: + # Standard accounts may not have access to the performance + # counters. + print_warning(f'Failed to create WindowsLoadTracker: {error}') + + def stop_load_tracker(self) -> None: + if self.win_load_tracker is None: + return + self.win_load_tracker.close() + self.win_load_tracker = None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 75c3d0e8350ade..74ef69b7c65307 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,6 +10,7 @@ import time import unittest from test.libregrtest.cmdline import _parse_args, Namespace +from test.libregrtest.logger import Logger from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, @@ -54,6 +55,8 @@ class Regrtest: on the command line. """ def __init__(self, ns: Namespace): + self.logger = Logger() + # Actions self.want_header: bool = ns.header self.want_list_tests: bool = ns.list_tests @@ -137,29 +140,8 @@ def __init__(self, ns: Namespace): self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - # misc - self.win_load_tracker = None - def log(self, line=''): - empty = not line - - # add the system load prefix: "load avg: 1.80 " - load_avg = self.getloadavg() - if load_avg is not None: - line = f"load avg: {load_avg:.2f} {line}" - - # add the timestamp prefix: "0:01:05 " - test_time = time.perf_counter() - self.start_time - - mins, secs = divmod(int(test_time), 60) - hours, mins = divmod(mins, 60) - test_time = "%d:%02d:%02d" % (hours, mins, secs) - - line = f"{test_time} {line}" - if empty: - line = line[:-1] - - print(line, flush=True) + self.logger.log(line) def display_progress(self, test_index, text): if self.quiet: @@ -293,7 +275,7 @@ def _rerun_failed_tests(self, runtests: RunTests): fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False) - self.set_tests(runtests) + self.logger.set_tests(runtests) self._run_tests_mp(runtests, self.num_workers) return runtests @@ -437,44 +419,7 @@ def get_state(self): def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from test.libregrtest.runtest_mp import RunWorkers - - # If we're on windows and this is the parent runner (not a worker), - # track the load average. - if sys.platform == 'win32': - from test.libregrtest.win_utils import WindowsLoadTracker - - try: - self.win_load_tracker = WindowsLoadTracker() - except PermissionError as error: - # Standard accounts may not have access to the performance - # counters. - print(f'Failed to create WindowsLoadTracker: {error}') - - try: - RunWorkers(self, runtests, num_workers).run() - finally: - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None - - def set_tests(self, runtests: RunTests): - self.tests = runtests.tests - if runtests.forever: - self.test_count_text = '' - self.test_count_width = 3 - else: - self.test_count_text = '/{}'.format(len(self.tests)) - self.test_count_width = len(self.test_count_text) - 1 - - def run_tests(self, runtests: RunTests): - self.first_runtests = runtests - self.set_tests(runtests) - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - tracer = None - else: - tracer = self.run_tests_sequentially(runtests) - return tracer + RunWorkers(self, runtests, num_workers).run() def finalize_tests(self, tracer): if self.next_single_filename: @@ -496,7 +441,7 @@ def finalize_tests(self, tracer): self.results.write_junit(self.junit_filename) def display_summary(self): - duration = time.perf_counter() - self.start_time + duration = time.perf_counter() - self.logger.start_time filtered = bool(self.match_tests) or bool(self.ignore_tests) # Total duration @@ -619,35 +564,8 @@ def main(self, tests: TestList | None = None): sys.exit(exc.code) - def getloadavg(self): - if self.win_load_tracker is not None: - return self.win_load_tracker.getloadavg() - - if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] - - return None - - def action_run_tests(self): - if self.hunt_refleak and self.hunt_refleak.warmups < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and " - "less than 3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - # For a partial run, we do not need to clutter the output. - if (self.want_header - or not(self.pgo or self.quiet or self.single_test_run - or self.tests or self.cmdline_args)): - self.display_header() - - if self.randomize: - print("Using random seed", self.random_seed) - - if self.num_workers < 0: - # Use all cores + extras for tests that like to sleep - self.num_workers = 2 + (os.cpu_count() or 1) - - runtests = RunTests( + def create_run_tests(self): + return RunTests( tuple(self.selected), fail_fast=self.fail_fast, match_tests=self.match_tests, @@ -668,17 +586,53 @@ def action_run_tests(self): python_cmd=self.python_cmd, ) + def run_tests(self) -> int: + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + if self.num_workers < 0: + # Use all CPUs + 2 extra worker processes for tests + # that like to sleep + self.num_workers = (os.cpu_count() or 1) + 2 + + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.pgo or self.quiet or self.single_test_run + or self.tests or self.cmdline_args)): + self.display_header() + + if self.randomize: + print("Using random seed", self.random_seed) + + runtests = self.create_run_tests() + self.first_runtests = runtests + self.logger.set_tests(runtests) + setup_tests(runtests) - tracer = self.run_tests(runtests) - self.display_result(runtests) + self.logger.start_load_tracker() + try: + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + tracer = None + else: + tracer = self.run_tests_sequentially(runtests) - if self.want_rerun and self.results.need_rerun(): - self.rerun_failed_tests(runtests) + self.display_result(runtests) + + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) + finally: + self.logger.stop_load_tracker() self.display_summary() self.finalize_tests(tracer) + return self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import worker_process @@ -697,9 +651,7 @@ def _main(self): elif self.want_list_cases: self.list_cases() else: - self.action_run_tests() - exitcode = self.results.get_exitcode(self.fail_env_changed, - self.fail_rerun) + exitcode = self.run_tests() sys.exit(exitcode) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 179b6104a8f79a..c4cffff57b14c4 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -433,7 +433,7 @@ def get_running(workers: list[WorkerThread]) -> list[str]: class RunWorkers: def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: self.results: TestResults = regrtest.results - self.log = regrtest.log + self.log = regrtest.logger.log self.display_progress = regrtest.display_progress self.num_workers = num_workers self.runtests = runtests diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 9a60a3d40b4c2c..57d85432bbfc95 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -6,6 +6,9 @@ from test import support +MS_WINDOWS = (sys.platform == 'win32') + + def format_duration(seconds): ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) From 92578919a60ebe2b8d6d42377f1e27479c156d65 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Sep 2023 08:09:25 +0300 Subject: [PATCH 593/598] gh-109174: Add support of SimpleNamespace in copy.replace() (GH-109175) --- Doc/library/types.rst | 2 ++ Lib/test/test_types.py | 27 ++++++++++++++++++ ...-09-09-09-05-41.gh-issue-109174.OJea5s.rst | 1 + Objects/namespaceobject.c | 28 +++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 82300afef0641e..875916be1049a3 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -504,6 +504,8 @@ Additional Utility Classes and Functions However, for a structured record type use :func:`~collections.namedtuple` instead. + :class:`!SimpleNamespace` objects are supported by :func:`copy.replace`. + .. versionadded:: 3.3 .. versionchanged:: 3.9 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index f2efee90dc0240..c6bff79f903828 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1900,6 +1900,33 @@ def test_pickle(self): self.assertEqual(ns, ns_roundtrip, pname) + def test_replace(self): + ns = types.SimpleNamespace(x=11, y=22) + + ns2 = copy.replace(ns) + self.assertEqual(ns2, ns) + self.assertIsNot(ns2, ns) + self.assertIs(type(ns2), types.SimpleNamespace) + self.assertEqual(vars(ns2), {'x': 11, 'y': 22}) + ns2.x = 3 + self.assertEqual(ns.x, 11) + ns.x = 4 + self.assertEqual(ns2.x, 3) + + self.assertEqual(vars(copy.replace(ns, x=1)), {'x': 1, 'y': 22}) + self.assertEqual(vars(copy.replace(ns, y=2)), {'x': 4, 'y': 2}) + self.assertEqual(vars(copy.replace(ns, x=1, y=2)), {'x': 1, 'y': 2}) + + def test_replace_subclass(self): + class Spam(types.SimpleNamespace): + pass + + spam = Spam(ham=8, eggs=9) + spam2 = copy.replace(spam, ham=5) + + self.assertIs(type(spam2), Spam) + self.assertEqual(vars(spam2), {'ham': 5, 'eggs': 9}) + def test_fake_namespace_compare(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # SystemError. diff --git a/Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst b/Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst new file mode 100644 index 00000000000000..63461fac3b96f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-09-09-05-41.gh-issue-109174.OJea5s.rst @@ -0,0 +1 @@ +Add support of :class:`types.SimpleNamespace` in :func:`copy.replace`. diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index 11cf859add3ab8..204c114fd9de2d 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -189,9 +189,37 @@ namespace_reduce(_PyNamespaceObject *ns, PyObject *Py_UNUSED(ignored)) } +static PyObject * +namespace_replace(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoPositional("__replace__", args)) { + return NULL; + } + + PyObject *result = PyObject_CallNoArgs((PyObject *)Py_TYPE(self)); + if (!result) { + return NULL; + } + if (PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, + ((_PyNamespaceObject*)self)->ns_dict) < 0) + { + Py_DECREF(result); + return NULL; + } + if (kwargs) { + if (PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, kwargs) < 0) { + Py_DECREF(result); + return NULL; + } + } + return result; +} + + static PyMethodDef namespace_methods[] = { {"__reduce__", (PyCFunction)namespace_reduce, METH_NOARGS, namespace_reduce__doc__}, + {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, NULL}, {NULL, NULL} // sentinel }; From 85a5d3dbe19249ba8d414d63328ae84241a35528 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sun, 10 Sep 2023 09:06:08 +0200 Subject: [PATCH 594/598] gh-93627: Align Python implementation of pickle with C implementation of pickle (GH-103035) If a method like __reduce_ex_ or __reduce__ is set to None, a TypeError is raised. --- Lib/pickle.py | 22 +++---- Lib/test/pickletester.py | 59 +++++++++++++++++++ ...3-03-26-19-11-10.gh-issue-93627.0UgwBL.rst | 1 + 3 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst diff --git a/Lib/pickle.py b/Lib/pickle.py index fe86f80f51d3b9..4f5ad5b71e8899 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -396,6 +396,8 @@ def decode_long(data): return int.from_bytes(data, byteorder='little', signed=True) +_NoValue = object() + # Pickling machinery class _Pickler: @@ -542,8 +544,8 @@ def save(self, obj, save_persistent_id=True): return rv = NotImplemented - reduce = getattr(self, "reducer_override", None) - if reduce is not None: + reduce = getattr(self, "reducer_override", _NoValue) + if reduce is not _NoValue: rv = reduce(obj) if rv is NotImplemented: @@ -556,8 +558,8 @@ def save(self, obj, save_persistent_id=True): # Check private dispatch table if any, or else # copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) - if reduce is not None: + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue) + if reduce is not _NoValue: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular @@ -567,12 +569,12 @@ def save(self, obj, save_persistent_id=True): return # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce_ex__", _NoValue) + if reduce is not _NoValue: rv = reduce(self.proto) else: - reduce = getattr(obj, "__reduce__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce__", _NoValue) + if reduce is not _NoValue: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % @@ -1705,8 +1707,8 @@ def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] - setstate = getattr(inst, "__setstate__", None) - if setstate is not None: + setstate = getattr(inst, "__setstate__", _NoValue) + if setstate is not _NoValue: setstate(state) return slotstate = None diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index a687fe0629080a..ddb180ef5ef825 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2408,6 +2408,22 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) + def test_reduce_ex_None(self): + c = REX_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_reduce_None(self): + c = R_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_pickle_setstate_None(self): + c = C_None_setstate() + p = self.dumps(c) + with self.assertRaises(TypeError): + self.loads(p) + @no_tracing def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ @@ -3349,6 +3365,21 @@ def __setstate__(self, state): def __reduce__(self): return type(self), (), self.state +class REX_None: + """ Setting __reduce_ex__ to None should fail """ + __reduce_ex__ = None + +class R_None: + """ Setting __reduce__ to None should fail """ + __reduce__ = None + +class C_None_setstate: + """ Setting __setstate__ to None should fail """ + def __getstate__(self): + return 1 + + __setstate__ = None + # Test classes for newobj @@ -3752,6 +3783,25 @@ def test_unpickling_buffering_readline(self): unpickler = self.unpickler_class(f) self.assertEqual(unpickler.load(), data) + def test_pickle_invalid_reducer_override(self): + # gh-103035 + obj = object() + + f = io.BytesIO() + class MyPickler(self.pickler_class): + pass + pickler = MyPickler(f) + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = None + with self.assertRaises(TypeError): + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = 10 + with self.assertRaises(TypeError): + pickler.dump(obj) # Tests for dispatch_table attribute @@ -3914,6 +3964,15 @@ def dumps(obj, protocol=None): self._test_dispatch_table(dumps, dt) + def test_dispatch_table_None_item(self): + # gh-93627 + obj = object() + f = io.BytesIO() + pickler = self.pickler_class(f) + pickler.dispatch_table = {type(obj): None} + with self.assertRaises(TypeError): + pickler.dump(obj) + def _test_dispatch_table(self, dumps, dispatch_table): def custom_load_dump(obj): return pickle.loads(dumps(obj, 0)) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst new file mode 100644 index 00000000000000..854da44b560b21 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-26-19-11-10.gh-issue-93627.0UgwBL.rst @@ -0,0 +1 @@ +Update the Python pickle module implementation to match the C implementation of the pickle module. For objects setting reduction methods like :meth:`~object.__reduce_ex__` or :meth:`~object.__reduce__` to ``None``, pickling will result in a :exc:`TypeError`. From 2dd6a86c4ee604b331ed739c2508b0d0114993c6 Mon Sep 17 00:00:00 2001 From: Delgan <4193924+Delgan@users.noreply.github.com> Date: Sun, 10 Sep 2023 12:55:56 +0200 Subject: [PATCH 595/598] Fix "FSTRING_MIDDLE" typo in py312 "What's New" (#109222) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index bdccbe1012c9e2..a46c913c4997ba 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1599,7 +1599,7 @@ Changes in the Python API functions is now changed due to the changes introduced in :pep:`701`. This means that ``STRING`` tokens are not emitted any more for f-strings and the tokens described in :pep:`701` are now produced instead: ``FSTRING_START``, - ``FSRING_MIDDLE`` and ``FSTRING_END`` are now emitted for f-string "string" + ``FSTRING_MIDDLE`` and ``FSTRING_END`` are now emitted for f-string "string" parts in addition to the appropriate tokens for the tokenization in the expression components. For example for the f-string ``f"start {1+1} end"`` the old version of the tokenizer emitted:: From 429749969621b149c1a7c3c004bd44f52bec8f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91line?= <31395137+yunline@users.noreply.github.com> Date: Sun, 10 Sep 2023 20:04:24 +0800 Subject: [PATCH 596/598] gh-109207: Fix SystemError when printing symtable entry object. (GH-109225) --- Lib/test/test_symtable.py | 4 ++++ .../2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst | 1 + Python/symtable.c | 5 ++--- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 25714aecda3a15..36cb7b3f242e4c 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -251,6 +251,10 @@ def test_symtable_repr(self): self.assertEqual(str(self.top), "") self.assertEqual(str(self.spam), "") + def test_symtable_entry_repr(self): + expected = f"" + self.assertEqual(repr(self.top._table), expected) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst new file mode 100644 index 00000000000000..f9da3ac4d1abbd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-10-18-53-55.gh-issue-109207.Fei8bY.rst @@ -0,0 +1 @@ +Fix a SystemError in ``__repr__`` of symtable entry object. diff --git a/Python/symtable.c b/Python/symtable.c index 7eef6f7231c035..6c28b49a0655c6 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -150,9 +150,8 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, static PyObject * ste_repr(PySTEntryObject *ste) { - return PyUnicode_FromFormat("", - ste->ste_name, - PyLong_AS_LONG(ste->ste_id), ste->ste_lineno); + return PyUnicode_FromFormat("", + ste->ste_name, ste->ste_id, ste->ste_lineno); } static void From 71b6e2602c0b5b06d51855f179cdb30094a67968 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Sep 2023 18:21:13 +0200 Subject: [PATCH 597/598] gh-109054: Don't use libatomic on cross-compilation (#109211) configure no longer uses libatomic by default when Python is cross-compiled. The LIBATOMIC variable can be set manually in this case: ./configure LIBATOMIC="-latomic" (...) --- configure | 7 ++----- configure.ac | 7 ++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/configure b/configure index c78c45d11260f6..8326a1db06c2da 100755 --- a/configure +++ b/configure @@ -27774,7 +27774,7 @@ then : else $as_nop if test "$cross_compiling" = yes then : - ac_cv_libatomic_needed=yes + ac_cv_libatomic_needed=no else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -27811,17 +27811,14 @@ int main() _ACEOF if ac_fn_c_try_run "$LINENO" then : - ac_cv_libatomic_needed=no - else $as_nop - ac_cv_libatomic_needed=yes + ac_cv_libatomic_needed=yes fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi - fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 printf "%s\n" "$ac_cv_libatomic_needed" >&6; } diff --git a/configure.ac b/configure.ac index f833755a466012..843f2b267a5253 100644 --- a/configure.ac +++ b/configure.ac @@ -7007,9 +7007,10 @@ int main() } return 0; // all good } -]])],[ - ac_cv_libatomic_needed=no -],[ac_cv_libatomic_needed=yes],[ac_cv_libatomic_needed=yes]) +]])], + [ac_cv_libatomic_needed=no], dnl build succeeded + [ac_cv_libatomic_needed=yes], dnl build failed + [ac_cv_libatomic_needed=no]) dnl cross compilation ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], From d6892c2b9263b39ea1c7905667942914b6a24b2c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Sep 2023 20:06:09 +0300 Subject: [PATCH 598/598] gh-50644: Forbid pickling of codecs streams (GH-109180) Attempts to pickle or create a shallow or deep copy of codecs streams now raise a TypeError. Previously, copying failed with a RecursionError, while pickling produced wrong results that eventually caused unpickling to fail with a RecursionError. --- Lib/codecs.py | 12 +++ Lib/test/test_codecs.py | 79 +++++++++++++++++++ ...3-09-09-15-08-37.gh-issue-50644.JUAZOh.rst | 4 + 3 files changed, 95 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst diff --git a/Lib/codecs.py b/Lib/codecs.py index c1c55d8afef389..82f23983e719c2 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -414,6 +414,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReader(Codec): @@ -663,6 +666,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReaderWriter: @@ -750,6 +756,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamRecoder: @@ -866,6 +875,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 91d7eaf997ae20..b5e9271ac0c3cd 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,7 +1,9 @@ import codecs import contextlib +import copy import io import locale +import pickle import sys import unittest import encodings @@ -1771,6 +1773,61 @@ def test_readlines(self): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + def test_copy(self): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + pickle.dumps(f, proto) + + +class StreamWriterTest(unittest.TestCase): + + def setUp(self): + self.writer = codecs.getwriter('utf-8') + + def test_copy(self): + f = self.writer(Queue(b'')) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = self.writer(Queue(b'')) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + pickle.dumps(f, proto) + + +class StreamReaderWriterTest(unittest.TestCase): + + def setUp(self): + self.reader = codecs.getreader('latin1') + self.writer = codecs.getwriter('utf-8') + + def test_copy(self): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + pickle.dumps(f, proto) + class EncodedFileTest(unittest.TestCase): @@ -3346,6 +3403,28 @@ def test_seeking_write(self): self.assertEqual(sr.readline(), b'abc\n') self.assertEqual(sr.readline(), b'789\n') + def test_copy(self): + bio = io.BytesIO() + codec = codecs.lookup('ascii') + sr = codecs.StreamRecoder(bio, codec.encode, codec.decode, + encodings.ascii.StreamReader, encodings.ascii.StreamWriter) + + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + copy.copy(sr) + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + copy.deepcopy(sr) + + def test_pickle(self): + q = Queue(b'') + codec = codecs.lookup('ascii') + sr = codecs.StreamRecoder(q, codec.encode, codec.decode, + encodings.ascii.StreamReader, encodings.ascii.StreamWriter) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + pickle.dumps(sr, proto) + @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class LocaleCodecTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst b/Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst new file mode 100644 index 00000000000000..a7a442e35289d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-09-15-08-37.gh-issue-50644.JUAZOh.rst @@ -0,0 +1,4 @@ +Attempts to pickle or create a shallow or deep copy of :mod:`codecs` streams +now raise a TypeError. Previously, copying failed with a RecursionError, +while pickling produced wrong results that eventually caused unpickling +to fail with a RecursionError.