From 2632610ec19123f7fb85effd632775011f92ab35 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 4 Sep 2026 15:19:47 +0200 Subject: [PATCH 1/8] gh-142349: Clarify that sys.lazy_modules may contain extra items (GH-155547) As an author of a debugging/introspection tool, I need an honest description of what's in `lazy_modules` and what kind of post-processing I'm expected to do. --- Doc/library/sys.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index bcc8f855c06f71..d5c38d6c05cc3b 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1488,11 +1488,24 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. data:: lazy_modules A :class:`set` of fully qualified module name strings that have been lazily - imported in the current interpreter but not yet loaded. When a - lazily imported module is accessed for the first time, its name is removed - from this set. + imported in the current interpreter but not yet loaded. + When a lazily imported module is accessed for the first time, its name is + typically removed from this set. - This attribute is intended for debugging and introspection. + The set may contain some additional strings. + It is intended for debugging and introspection, and consumers are expected + to verify each entry's status. + + .. impl-detail:: + + Currently, :data:`!lazy_modules` may also contain: + + * names of *attributes* (non-modules), such as ``"pathlib.Path"`` after + running ``lazy from pathlib import Path``, and + * names of items than have already been accessed. + + In future versions of Python, these may be removed, and/or additional + extras may be added. See also :func:`set_lazy_imports` and :pep:`810`. From 7d71b3eae3cc297af3a55a30fccc85da5adeea7a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 4 Sep 2026 15:21:01 +0200 Subject: [PATCH 2/8] gh-97850: Add load_module() removal to 3.15 What's New (GH-156418) --- Doc/whatsnew/3.15.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index bf9aceaa20b2b4..118d9b3e32b28f 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -2114,6 +2114,17 @@ http.server (Contributed by Bénédikt Tran in :gh:`133810`.) +importlib +--------- + +* The ``load_module()`` methods of :class:`~importlib.abc.Loader` and its + subclasses is removed. + The import system will no longer call it when defined on custom subclasses. + The method has been deprecated in favor of + :meth:`~importlib.abc.Loader.exec_module` since Python 3.4. + (Contributed by Brett Cannon in :gh:`97850`.) + + importlib.resources ------------------- From aaae15c35b5fba1246b8998e6586a76fc1541be7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 4 Sep 2026 16:44:57 +0300 Subject: [PATCH 3/8] gh-155907: Fix error handling in the marshal C API (GH-155909) Functions reading marshalled data from a FILE* now raise OSError for I/O errors and KeyboardInterrupt for interrupted reading, instead of EOFError. PyMarshal_WriteObjectToFile() and PyMarshal_WriteLongToFile() now detect I/O errors and interrupted writing instead of ignoring them. PyMarshal_WriteObjectToFile() now also sets the error indicator if the value cannot be marshalled. Co-authored-by: Claude Opus 5 (1M context) --- Doc/c-api/marshal.rst | 31 ++-- Doc/whatsnew/3.16.rst | 14 ++ Lib/test/test_marshal.py | 29 ++++ ...-08-16-20-48-08.gh-issue-155907.Vb3xTn.rst | 7 + Modules/_testcapimodule.c | 8 +- Python/marshal.c | 142 +++++++++++++----- 6 files changed, 183 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-08-16-20-48-08.gh-issue-155907.Vb3xTn.rst diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 668a163b2df5a1..5bf4757f8ae158 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -16,6 +16,20 @@ Numeric values are stored with the least significant byte first. The module supports several versions of the data format; see the :py:mod:`Python module documentation ` for details. +The following exceptions can be raised by these functions: +:exc:`ValueError` if the value cannot be marshalled, +:exc:`ValueError` or :exc:`TypeError` if the data is malformed, +:exc:`EOFError` if the end of the data is reached before the value is complete, +:exc:`OSError` if reading from or writing to a :c:expr:`FILE*` fails, +:exc:`KeyboardInterrupt` if reading or writing is interrupted by a signal, +and :exc:`MemoryError` if memory allocation fails. + +.. versionchanged:: next + Previously, in functions taking a :c:expr:`FILE*`, + the reading functions raised :exc:`EOFError` + instead of :exc:`OSError` and :exc:`KeyboardInterrupt`, + and the writing functions ignored I/O errors and interruptions. + .. c:macro:: Py_MARSHAL_VERSION The current format version. See :py:data:`marshal.version`. @@ -42,6 +56,8 @@ the :py:mod:`Python module documentation ` for details. Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. + On error, raises an exception and returns ``NULL``. + The following functions allow marshalled values to be read back in. @@ -52,8 +68,7 @@ The following functions allow marshalled values to be read back in. for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:expr:`long`. - On error, sets the appropriate exception (:exc:`EOFError`) and returns - ``-1``. + On error, raises an exception and returns ``-1``. .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) @@ -62,8 +77,7 @@ The following functions allow marshalled values to be read back in. for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:expr:`short`. - On error, sets the appropriate exception (:exc:`EOFError`) and returns - ``-1``. + On error, raises an exception and returns ``-1``. .. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) @@ -71,8 +85,7 @@ The following functions allow marshalled values to be read back in. Return a Python object from the data stream in a :c:expr:`FILE*` opened for reading. - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns ``NULL``. + On error, raises an exception and returns ``NULL``. .. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) @@ -85,8 +98,7 @@ The following functions allow marshalled values to be read back in. file. Only use this variant if you are certain that you won't be reading anything else from the file. - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns ``NULL``. + On error, raises an exception and returns ``NULL``. .. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) @@ -94,6 +106,5 @@ The following functions allow marshalled values to be read back in. Return a Python object from the data stream in a byte buffer containing *len* bytes pointed to by *data*. - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns ``NULL``. + On error, raises an exception and returns ``NULL``. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 3262acd87d6d49..75541f1e106752 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -956,6 +956,20 @@ Porting to Python 3.16 * :c:func:`PyType_ClearCache` is now a no-op as the type cache is now implemented per-type. It still returns the current version tag. +* Functions reading marshalled data from a :c:expr:`FILE*`, + such as :c:func:`PyMarshal_ReadObjectFromFile`, + now raise :exc:`OSError` for I/O errors + and :exc:`KeyboardInterrupt` for interrupted reading, + instead of :exc:`EOFError`. + (Contributed by Serhiy Storchaka in :gh:`155907`.) + +* :c:func:`PyMarshal_WriteLongToFile` and :c:func:`PyMarshal_WriteObjectToFile` + now set the error indicator for I/O errors and interrupted writing, + instead of ignoring them. + :c:func:`PyMarshal_WriteObjectToFile` now also sets the error indicator + if the value cannot be marshalled. + (Contributed by Serhiy Storchaka in :gh:`155907`.) + Deprecated C APIs ----------------- diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index b5bacadbfd381f..c595e8cf14f1e1 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -793,6 +793,35 @@ def test_slice(self): @unittest.skipUnless(_testcapi, 'requires _testcapi') class CAPI_TestCase(unittest.TestCase, HelperMixin): + def test_read_from_file_error(self): + # A read error is reported as OSError, not EOFError. + # A directory cannot be read (on some platforms it cannot even + # be opened, which is reported as OSError as well). + os.mkdir(os_helper.TESTFN) + self.addCleanup(os_helper.rmdir, os_helper.TESTFN) + for func in (_testcapi.pymarshal_read_short_from_file, + _testcapi.pymarshal_read_long_from_file, + _testcapi.pymarshal_read_object_from_file, + _testcapi.pymarshal_read_last_object_from_file): + with self.subTest(func=func.__name__): + self.assertRaises(OSError, func, os_helper.TESTFN) + + @unittest.skipUnless(os.path.exists('/dev/full'), 'requires /dev/full') + def test_write_to_file_error(self): + # A write error is reported as OSError. + # The data is large enough to not fit in the stdio buffer, so that + # the error is detected before the file is closed. + obj = b'x' * 100000 + with self.assertRaises(OSError): + _testcapi.pymarshal_write_object_to_file(obj, '/dev/full', + marshal.version) + + def test_write_unmarshallable_to_file(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with self.assertRaisesRegex(ValueError, 'unmarshallable object'): + _testcapi.pymarshal_write_object_to_file(object(), os_helper.TESTFN, + marshal.version) + def test_write_long_to_file(self): for v in range(marshal.version + 1): _testcapi.pymarshal_write_long_to_file(0x12345678, os_helper.TESTFN, v) diff --git a/Misc/NEWS.d/next/C_API/2026-08-16-20-48-08.gh-issue-155907.Vb3xTn.rst b/Misc/NEWS.d/next/C_API/2026-08-16-20-48-08.gh-issue-155907.Vb3xTn.rst new file mode 100644 index 00000000000000..215bb2d031c42e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-16-20-48-08.gh-issue-155907.Vb3xTn.rst @@ -0,0 +1,7 @@ +:c:func:`PyMarshal_ReadObjectFromFile` and other functions reading marshalled +data from a :c:expr:`FILE*` now raise :exc:`OSError` for I/O errors and +:exc:`KeyboardInterrupt` for interrupted reading, instead of :exc:`EOFError`. +:c:func:`PyMarshal_WriteObjectToFile` and :c:func:`PyMarshal_WriteLongToFile` +now detect I/O errors and interrupted writing instead of ignoring them. +:c:func:`PyMarshal_WriteObjectToFile` now also sets the error indicator if the +value cannot be marshalled. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f6009e7e73f249..0312ee9066231c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1436,9 +1436,11 @@ pymarshal_write_long_to_file(PyObject* self, PyObject *args) } PyMarshal_WriteLongToFile(value, fp, version); - assert(!PyErr_Occurred()); fclose(fp); + if (PyErr_Occurred()) { + return NULL; + } Py_RETURN_NONE; } @@ -1460,9 +1462,11 @@ pymarshal_write_object_to_file(PyObject* self, PyObject *args) } PyMarshal_WriteObjectToFile(obj, fp, version); - assert(!PyErr_Occurred()); fclose(fp); + if (PyErr_Occurred()) { + return NULL; + } Py_RETURN_NONE; } diff --git a/Python/marshal.c b/Python/marshal.c index ef5a8d3840cd80..1897d700c055bd 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -106,6 +106,7 @@ module marshal #define WFERR_NESTEDTOODEEP 2 #define WFERR_NOMEMORY 3 #define WFERR_CODE_NOT_ALLOWED 4 +#define WFERR_EXCEPTION_SET 5 /* An exception has already been raised. */ typedef struct { FILE *fp; @@ -125,11 +126,32 @@ typedef struct { *(p)->ptr++ = (c); \ } while(0) +/* Report a failure of the underlying file. An earlier error is not + overwritten. */ +static void +w_file_error(WFILE *p) +{ + int saved_errno = errno; + if (p->error != WFERR_OK) { + return; + } + p->error = WFERR_EXCEPTION_SET; + if (PyErr_CheckSignals()) { + /* The signal handler has raised an exception. */ + return; + } + errno = saved_errno; + PyErr_SetFromErrno(PyExc_OSError); +} + static void w_flush(WFILE *p) { assert(p->fp != NULL); - fwrite(p->buf, 1, p->ptr - p->buf, p->fp); + size_t n = (size_t)(p->ptr - p->buf); + if (fwrite(p->buf, 1, n, p->fp) != n) { + w_file_error(p); + } p->ptr = p->buf; } @@ -182,7 +204,9 @@ w_string(const void *s, Py_ssize_t n, WFILE *p) } else { w_flush(p); - fwrite(s, 1, n, p->fp); + if (fwrite(s, 1, n, p->fp) != (size_t)n) { + w_file_error(p); + } } } else { @@ -782,11 +806,36 @@ w_clear_refs(WFILE *wf) } } +/* Set the exception indicator according to the recorded error. */ +static void +w_set_exception(WFILE *p) +{ + assert(p->error != WFERR_OK); + switch (p->error) { + case WFERR_NOMEMORY: + PyErr_NoMemory(); + break; + case WFERR_NESTEDTOODEEP: + PyErr_SetString(PyExc_ValueError, + "object too deeply nested to marshal"); + break; + case WFERR_CODE_NOT_ALLOWED: + PyErr_SetString(PyExc_ValueError, + "marshalling code objects is disallowed"); + break; + case WFERR_EXCEPTION_SET: + /* An exception has already been raised. */ + assert(PyErr_Occurred()); + break; + default: + case WFERR_UNMARSHALLABLE: + PyErr_SetString(PyExc_ValueError, + "unmarshallable object"); + break; + } +} + /* version currently has no effect for writing ints. */ -/* Note that while the documentation states that this function - * can error, currently it never does. Setting an exception in - * this function should be regarded as an API-breaking change. - */ void PyMarshal_WriteLongToFile(long x, FILE *fp, int version) { @@ -800,6 +849,9 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) wf.version = version; w_long(x, &wf); w_flush(&wf); + if (wf.error != WFERR_OK) { + w_set_exception(&wf); + } } void @@ -823,6 +875,9 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) w_object(x, &wf); w_clear_refs(&wf); w_flush(&wf); + if (wf.error != WFERR_OK) { + w_set_exception(&wf); + } } typedef struct { @@ -875,6 +930,14 @@ r_string(Py_ssize_t n, RFILE *p) if (!p->readable) { assert(p->fp != NULL); read = fread(p->buf, 1, n, p->fp); + if (read != n) { + assert(read < n); + int saved_errno = errno; + if (!PyErr_CheckSignals() && ferror(p->fp)) { + errno = saved_errno; + PyErr_SetFromErrno(PyExc_OSError); + } + } } else { PyObject *res, *mview; @@ -887,21 +950,26 @@ r_string(Py_ssize_t n, RFILE *p) return NULL; res = _PyObject_CallMethod(p->readable, &_Py_ID(readinto), "N", mview); - if (res != NULL) { - read = PyNumber_AsSsize_t(res, PyExc_ValueError); - Py_DECREF(res); + if (res == NULL) { + return NULL; + } + read = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_DECREF(res); + if (read == -1 && PyErr_Occurred()) { + return NULL; + } + if (read > n) { + PyErr_Format(PyExc_ValueError, + "read() returned too much data: " + "%zd bytes requested, %zd returned", + n, read); + return NULL; } } if (read != n) { if (!PyErr_Occurred()) { - if (read > n) - PyErr_Format(PyExc_ValueError, - "read() returned too much data: " - "%zd bytes requested, %zd returned", - n, read); - else - PyErr_SetString(PyExc_EOFError, - "EOF read where not expected"); + PyErr_SetString(PyExc_EOFError, + "EOF read where not expected"); } return NULL; } @@ -922,6 +990,15 @@ r_byte(RFILE *p) if (c != EOF) { return c; } + int saved_errno = errno; + if (PyErr_CheckSignals()) { + return EOF; + } + if (ferror(p->fp)) { + errno = saved_errno; + PyErr_SetFromErrno(PyExc_OSError); + return EOF; + } } else { const char *ptr = r_string(1, p); @@ -1850,8 +1927,18 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp) if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) { char* pBuf = (char *)PyMem_Malloc(filesize); if (pBuf != NULL) { + PyObject *v = NULL; size_t n = fread(pBuf, 1, (size_t)filesize, fp); - PyObject* v = PyMarshal_ReadObjectFromString(pBuf, n); + int saved_errno = errno; + if (!PyErr_CheckSignals()) { + if (ferror(fp)) { + errno = saved_errno; + PyErr_SetFromErrno(PyExc_OSError); + } + else { + v = PyMarshal_ReadObjectFromString(pBuf, n); + } + } PyMem_Free(pBuf); return v; } @@ -1938,24 +2025,7 @@ _PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) } if (wf.error != WFERR_OK) { Py_XDECREF(wf.str); - switch (wf.error) { - case WFERR_NOMEMORY: - PyErr_NoMemory(); - break; - case WFERR_NESTEDTOODEEP: - PyErr_SetString(PyExc_ValueError, - "object too deeply nested to marshal"); - break; - case WFERR_CODE_NOT_ALLOWED: - PyErr_SetString(PyExc_ValueError, - "marshalling code objects is disallowed"); - break; - default: - case WFERR_UNMARSHALLABLE: - PyErr_SetString(PyExc_ValueError, - "unmarshallable object"); - break; - } + w_set_exception(&wf); return NULL; } return wf.str; From bc977f450b328656e073b547f23aa201401023c7 Mon Sep 17 00:00:00 2001 From: Tomasz Kazimierczak Date: Fri, 4 Sep 2026 16:20:30 +0200 Subject: [PATCH 4/8] gh-118150: difflib: expose autojunk flag from SequenceMatcher to public methods and functions (GH-153959) --- Doc/library/difflib.rst | 59 +++++++++++-- Doc/whatsnew/3.16.rst | 31 +++++++ Lib/difflib.py | 46 ++++++---- Lib/test/test_difflib.py | 84 ++++++++++++++++++- ...-07-18-16-43-02.gh-issue-118150.x0mV8P.rst | 5 ++ 5 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 5339186b72f6bc..583bf78d1649b6 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -128,7 +128,7 @@ Diff generation The :class:`Differ` class has this constructor: - .. method:: __init__(linejunk=None, charjunk=None) + .. method:: __init__(linejunk=None, charjunk=None, autojunk=True) Optional keyword parameters *linejunk* and *charjunk* are for filter functions (or ``None``): @@ -147,6 +147,14 @@ Diff generation :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* parameter for an explanation. + Setting the optional *autojunk* argument to ``False`` will turn + :ref:`automatic junk heuristic ` off. + + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + + + :class:`Differ` objects are used (deltas generated) via a single method: @@ -161,6 +169,8 @@ Diff generation printed as-is via the :meth:`~io.IOBase.writelines` method of a file-like object. + + .. class:: HtmlDiff This class can be used to create an HTML table (or a complete HTML file @@ -176,7 +186,7 @@ Diff generation The constructor for this class is: - .. method:: __init__(tabsize=8, wrapcolumn=None, linejunk=None, charjunk=IS_CHARACTER_JUNK) + .. method:: __init__(tabsize=8, wrapcolumn=None, linejunk=None, charjunk=IS_CHARACTER_JUNK, *, autojunk=True) Initializes instance of :class:`HtmlDiff`. @@ -187,8 +197,15 @@ Diff generation broken and wrapped, defaults to ``None`` where lines are not wrapped. *linejunk* and *charjunk* are optional keyword arguments passed into :func:`ndiff` - (used by :class:`HtmlDiff` to generate the side by side HTML differences). See - :func:`ndiff` documentation for argument default values and descriptions. + (used by :class:`HtmlDiff` to generate the side by side HTML differences). + See :func:`ndiff` documentation for argument default values and descriptions. + + Setting the optional *autojunk* argument to ``False`` will turn + :ref:`automatic junk heuristic ` off. + + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + The following methods are public: @@ -231,7 +248,7 @@ Diff generation -.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, autojunk=True) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in context diff format. @@ -277,8 +294,14 @@ Diff generation See :ref:`difflib-interface` for a more detailed example. + Setting the optional *autojunk* argument to ``False`` will turn + :ref:`automatic junk heuristic ` off. + + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. -.. function:: get_close_matches(word, possibilities, n=3, cutoff=0.6) + +.. function:: get_close_matches(word, possibilities, n=3, cutoff=0.6, *, autojunk=True) Return a list of the best "good enough" matches. *word* is a sequence for which close matches are desired (typically a string), and *possibilities* is a list of @@ -290,6 +313,9 @@ Diff generation Optional argument *cutoff* (default ``0.6``) is a float in the range [0, 1]. Possibilities that don't score at least that similar to *word* are ignored. + Setting the optional *autojunk* argument to ``False`` will turn + :ref:`automatic junk heuristic ` off. + The best (no more than *n*) matches among the possibilities are returned in a list, sorted by similarity score, most similar first. @@ -303,8 +329,11 @@ Diff generation >>> get_close_matches('accept', keyword.kwlist) ['except'] + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + -.. function:: ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK) +.. function:: ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK, *, autojunk=True) Compare *a* and *b* (lists of strings); return a :class:`Differ`\ -style delta (a :term:`generator` generating the delta lines). @@ -325,6 +354,11 @@ Diff generation function :func:`IS_CHARACTER_JUNK`, which filters out whitespace characters (a blank or tab; it's a bad idea to include newline in this!). + Setting the optional *autojunk* argument to ``False`` will turn + :ref:`automatic junk heuristic ` off. + + Example: + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> print(''.join(diff), end="") @@ -338,6 +372,9 @@ Diff generation + tree + emu + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + .. function:: restore(sequence, which) @@ -362,7 +399,7 @@ Diff generation emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, autojunk=True, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -410,6 +447,12 @@ Diff generation .. versionchanged:: 3.15 Added the *color* parameter. + Setting the optional *autojunk* argument to ``False`` will turn + :ref:`automatic junk heuristic ` off. + + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 75541f1e106752..858dc3b8a878e7 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -304,6 +304,37 @@ curses (Contributed by Serhiy Storchaka in :gh:`133031`.) +ctypes +------ + +* Add :func:`ctypes.util.struct` for generating :class:`~ctypes.Structure` types + from an annotation-based syntax, similar to how the :mod:`dataclasses` module + is used. + (Contributed by Peter Bierma in :gh:`104533`.) +* Add :func:`ctypes.util.wrap_dll_function` for generating function pointers + through a function signature. + (Contributed by Peter Bierma in :gh:`153903`.) + + +concurrent.futures +------------------ + +* The iterator returned by :meth:`concurrent.futures.Executor.map` is no longer + automatically closed if a function call raises an exception. + Use method :meth:`!close` to explicitly close the iterator. + (Contributed by xzmeng and Serhiy Storchaka in :gh:`108518`.) + + +difflib +------- + +* Expose optional ``autojunk`` parameter from :class:`difflib.SequenceMatcher` + to public functions and class methods in :mod:`difflib`, + allowing to modify behavior of automatic junk heuristic in this module + in higher public class methods and functions. + (Contributed by Tomasz Kazimierczak in :gh:`118150`) + + encodings --------- diff --git a/Lib/difflib.py b/Lib/difflib.py index 95ba8fd782c6c3..c081cd8606df5b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -664,7 +664,7 @@ def real_quick_ratio(self): __class_getitem__ = classmethod(GenericAlias) -def get_close_matches(word, possibilities, n=3, cutoff=0.6): +def get_close_matches(word, possibilities, n=3, cutoff=0.6, *, autojunk=True): """Use SequenceMatcher to return list of the best "good enough" matches. word is a sequence for which close matches are desired (typically a @@ -698,7 +698,7 @@ def get_close_matches(word, possibilities, n=3, cutoff=0.6): if not 0.0 <= cutoff <= 1.0: raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) result = [] - s = SequenceMatcher() + s = SequenceMatcher(autojunk=autojunk) s.set_seq2(word) for x in possibilities: s.set_seq1(x) @@ -810,7 +810,7 @@ class Differ: + 5. Flat is better than nested. """ - def __init__(self, linejunk=None, charjunk=None): + def __init__(self, linejunk=None, charjunk=None, *, autojunk=True): """ Construct a text differencer, with optional filters. @@ -828,10 +828,13 @@ def __init__(self, linejunk=None, charjunk=None): module-level function `IS_CHARACTER_JUNK` may be used to filter out whitespace characters (a blank or tab; **note**: bad idea to include newline in this!). Use of IS_CHARACTER_JUNK is recommended. + - `autojunk`: automatic junk diff heuristic + (refer to :class:`SequenceMatcher` for specifics). """ self.linejunk = linejunk self.charjunk = charjunk + self.autojunk = autojunk def compare(self, a, b): r""" @@ -859,7 +862,7 @@ def compare(self, a, b): + emu """ - cruncher = SequenceMatcher(self.linejunk, a, b) + cruncher = SequenceMatcher(self.linejunk, a, b, autojunk=self.autojunk) for tag, alo, ahi, blo, bhi in cruncher.get_opcodes(): if tag == 'replace': g = self._fancy_replace(a, alo, ahi, b, blo, bhi) @@ -920,7 +923,7 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): # Later, more pathological cases prompted removing recursion # entirely. cutoff = 0.74999 - cruncher = SequenceMatcher(self.charjunk) + cruncher = SequenceMatcher(self.charjunk, autojunk=self.autojunk) crqr = cruncher.real_quick_ratio cqr = cruncher.quick_ratio cr = cruncher.ratio @@ -1099,7 +1102,7 @@ def _format_range_unified(start, stop): return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n', *, color=False): + tofiledate='', n=3, lineterm='\n', *, autojunk=True, color=False): r""" Compare two sequences of lines; generate the delta as a unified diff. @@ -1120,6 +1123,9 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', 'git diff --color'. Even if enabled, it can be controlled using environment variables such as 'NO_COLOR'. + Set `autojunk` to False if you don't want automated junk heuristic. + See details in :class:`SequenceMatcher. + The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. @@ -1150,7 +1156,7 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False - for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): + for group in SequenceMatcher(None, a, b, autojunk=autojunk).get_grouped_opcodes(n): if not started: started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' @@ -1193,7 +1199,7 @@ def _format_range_context(start, stop): # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', - fromfiledate='', tofiledate='', n=3, lineterm='\n'): + fromfiledate='', tofiledate='', n=3, lineterm='\n', *, autojunk=True): r""" Compare two sequences of lines; generate the delta as a context diff. @@ -1216,6 +1222,10 @@ def context_diff(a, b, fromfile='', tofile='', The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. + The kwarg `autojunk` sets up automated junk heuristic with + :class:`SequenceMatcher`, which is used under the hood in this function. + See documentation of :class:`SequenceMatcher` for details. + Example: >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), @@ -1239,7 +1249,7 @@ def context_diff(a, b, fromfile='', tofile='', _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False - for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): + for group in SequenceMatcher(None, a, b, autojunk=autojunk).get_grouped_opcodes(n): if not started: started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' @@ -1321,7 +1331,7 @@ def decode(s): for line in lines: yield line.encode('ascii', 'surrogateescape') -def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): +def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK, *, autojunk=True): r""" Compare `a` and `b` (lists of strings); return a `Differ`-style delta. @@ -1339,6 +1349,8 @@ def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): whitespace characters (a blank or tab; note: it's a bad idea to include newline in this!). + - autojunk: automatic junk heuristic - refer to :class:`SequenceMatcher` for details + Tools/scripts/ndiff.py is a command-line front-end to this function. Example: @@ -1356,10 +1368,10 @@ def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): + tree + emu """ - return Differ(linejunk, charjunk).compare(a, b) + return Differ(linejunk, charjunk, autojunk=autojunk).compare(a, b) def _mdiff(fromlines, tolines, context=None, linejunk=None, - charjunk=IS_CHARACTER_JUNK): + charjunk=IS_CHARACTER_JUNK, *, autojunk=True): r"""Returns generator yielding marked up from/to side by side differences. Arguments: @@ -1369,6 +1381,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, if None, all from/to text lines will be generated. linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) + autojunk -- passed on to ndiff (see ndiff documentation) This function returns an iterator which returns a tuple: (from line tuple, to line tuple, boolean flag) @@ -1398,7 +1411,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, change_re = re.compile(r'(\++|\-+|\^+)') # create the difference iterator to generate the differences - diff_lines_iterator = ndiff(fromlines,tolines,linejunk,charjunk) + diff_lines_iterator = ndiff(fromlines, tolines, linejunk, charjunk, autojunk=autojunk) def _make_line(lines, format_key, side, num_lines=[0,0]): """Returns line of text with user's change markup and line formatting. @@ -1738,14 +1751,14 @@ class HtmlDiff(object): _default_prefix = 0 def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, - charjunk=IS_CHARACTER_JUNK): + charjunk=IS_CHARACTER_JUNK, *, autojunk=True): """HtmlDiff instance initializer Arguments: tabsize -- tab stop spacing, defaults to 8. wrapcolumn -- column number where lines are broken and wrapped, defaults to None where lines are not wrapped. - linejunk,charjunk -- keyword arguments passed into ndiff() (used by + linejunk, charjunk, autojunk -- keyword arguments passed into ndiff() (used by HtmlDiff() to generate the side by side HTML differences). See ndiff() documentation for argument default values and descriptions. """ @@ -1753,6 +1766,7 @@ def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, self._wrapcolumn = wrapcolumn self._linejunk = linejunk self._charjunk = charjunk + self._autojunk = autojunk def make_file(self, fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5, *, charset='utf-8'): @@ -2026,7 +2040,7 @@ def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, else: context_lines = None diffs = _mdiff(fromlines,tolines,context_lines,linejunk=self._linejunk, - charjunk=self._charjunk) + charjunk=self._charjunk, autojunk=self._autojunk) # set up iterator to wrap lines that exceed desired width if self._wrapcolumn: diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 4f99b7c91c654e..5babe834e9beac 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -56,7 +56,7 @@ def test_bjunk(self): class TestAutojunk(unittest.TestCase): - """Tests for the autojunk parameter added in 2.7""" + """Tests for the autojunk parameter added in SequenceMatcher and higher-level difflib APIs""" def test_one_insert_homogenous_sequence(self): # By default autojunk=True and the heuristic kicks in for a sequence # of length 200+ @@ -72,6 +72,88 @@ def test_one_insert_homogenous_sequence(self): self.assertAlmostEqual(sm.ratio(), 0.9975, places=3) self.assertEqual(sm.bpopular, set()) + def test_get_close_matches(self): + word = 'a' + 'b' * 200 + possibilities = ['b' * 200] + + # By default autojunk=True, so 'b' is junk -> ratio ~ 0 -> no matches + self.assertEqual(difflib.get_close_matches(word, possibilities, cutoff=0.6), []) + self.assertEqual(difflib.get_close_matches(word, possibilities, cutoff=0.6, autojunk=True), []) + + # With autojunk=False, ratio ~ 0.9975 -> match returned + self.assertEqual(difflib.get_close_matches(word, possibilities, cutoff=0.6, autojunk=False), ['b' * 200]) + + def test_differ_and_ndiff(self): + lines1 = ["x\n"] * 200 + ["a\n", "b\n", "c\n"] + ["x\n"] * 50 + lines2 = ["a\n", "b\n", "c\n"] + ["x\n"] * 250 + + # Line-level autojunk propagation + d_true = difflib.Differ(autojunk=True) + d_false = difflib.Differ(autojunk=False) + res_true = list(d_true.compare(lines1, lines2)) + res_false = list(d_false.compare(lines1, lines2)) + self.assertNotEqual(res_true, res_false) + + ndiff_true = list(difflib.ndiff(lines1, lines2, autojunk=True)) + ndiff_false = list(difflib.ndiff(lines1, lines2, autojunk=False)) + self.assertNotEqual(ndiff_true, ndiff_false) + self.assertEqual(ndiff_true, res_true) + self.assertEqual(ndiff_false, res_false) + + # Character-level autojunk propagation in Differ (_fancy_replace) + line1 = "x" * 200 + "abc" + "x" * 50 + "\n" + line2 = "abc" + "x" * 250 + "\n" + fancy_true = list(difflib.Differ(autojunk=True).compare([line1], [line2])) + fancy_false = list(difflib.Differ(autojunk=False).compare([line1], [line2])) + self.assertNotEqual(fancy_true, fancy_false) + + def test_unified_and_context_diff(self): + lines1 = ["x\n"] * 200 + ["a\n", "b\n", "c\n"] + ["x\n"] * 50 + lines2 = ["a\n", "b\n", "c\n"] + ["x\n"] * 250 + + u_true = list(difflib.unified_diff(lines1, lines2, autojunk=True)) + u_false = list(difflib.unified_diff(lines1, lines2, autojunk=False)) + self.assertNotEqual(u_true, u_false) + + c_true = list(difflib.context_diff(lines1, lines2, autojunk=True)) + c_false = list(difflib.context_diff(lines1, lines2, autojunk=False)) + self.assertNotEqual(c_true, c_false) + + def test_htmldiff(self): + lines1 = ["x\n"] * 200 + ["a\n", "b\n", "c\n"] + ["x\n"] * 50 + lines2 = ["a\n", "b\n", "c\n"] + ["x\n"] * 250 + + old_prefix = difflib.HtmlDiff._default_prefix + try: + html_true = difflib.HtmlDiff(autojunk=True).make_file(lines1, lines2) + html_false = difflib.HtmlDiff(autojunk=False).make_file(lines1, lines2) + self.assertNotEqual(html_true, html_false) + finally: + difflib.HtmlDiff._default_prefix = old_prefix + + def test_autojunk_signatures(self): + import inspect + + funcs = [ + difflib.get_close_matches, + difflib.unified_diff, + difflib.context_diff, + difflib.ndiff, + ] + for func in funcs: + sig = inspect.signature(func) + self.assertIn('autojunk', sig.parameters) + param = sig.parameters['autojunk'] + self.assertEqual(param.default, True) + self.assertEqual(param.kind, inspect.Parameter.KEYWORD_ONLY) + + for cls in [difflib.Differ, difflib.HtmlDiff]: + sig = inspect.signature(cls.__init__) + self.assertIn('autojunk', sig.parameters) + param = sig.parameters['autojunk'] + self.assertEqual(param.default, True) + + class TestSFbugs(unittest.TestCase): def test_ratio_for_null_seqn(self): diff --git a/Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst b/Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst new file mode 100644 index 00000000000000..b479302ade4064 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst @@ -0,0 +1,5 @@ +Expose automated junk heuristic kwarg-only flag ``autojunk`` from +:class:`difflib.SequenceMatcher` to the public functions +and class methods in the :mod:`difflib`. +See :class:`difflib.SequenceMatcher` documentation for details +and issue :gh:`118150` for the motivation. From 1860130962b2fd92274a1c2bc74dd459313a2341 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 4 Sep 2026 15:44:09 +0100 Subject: [PATCH 5/8] gh-156583: Remove unnecessary `test_sundry` (#156642) --- Lib/test/test_sundry.py | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 Lib/test/test_sundry.py diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py deleted file mode 100644 index d6d08ee53f821c..00000000000000 --- a/Lib/test/test_sundry.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Do a minimal test of all the modules that aren't otherwise tested.""" -import importlib -from test import support -from test.support import import_helper -from test.support import warnings_helper -import unittest - -class TestUntestedModules(unittest.TestCase): - def test_untested_modules_can_be_imported(self): - untested = ('encodings',) - with warnings_helper.check_warnings(quiet=True): - for name in untested: - try: - import_helper.import_module('test.test_{}'.format(name)) - except unittest.SkipTest: - importlib.import_module(name) - else: - self.fail('{} has tests even though test_sundry claims ' - 'otherwise'.format(name)) - - import html.entities # noqa: F401 - - try: - # Not available on Windows - import tty # noqa: F401 - except ImportError: - if support.verbose: - print("skipping tty") - - -if __name__ == "__main__": - unittest.main() From 1a2e3a034df6074a900bd9f2cf23c5b164f5320b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Fri, 4 Sep 2026 16:58:12 +0200 Subject: [PATCH 6/8] gh-140870: Add PyREPL's module attributes import completion feature to What's New 3.15 (#156782) Co-authored-by: Stan Ulbrych --- Doc/whatsnew/3.15.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 118d9b3e32b28f..4095058961b735 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -902,6 +902,13 @@ Default interactive shell `. (Contributed by Antonio Cuni and Pablo Galindo in :gh:`130472`.) +* Tab completion now suggests module attributes in ``from ... import`` statements. + Attributes can only be suggested once the module is imported, so + :term:`stdlib` modules are imported automatically, while for + other modules the completer offers to import them when :kbd:`Tab` + is pressed a second time. + (Contributed by Loïc Simon and Pablo Galindo in :gh:`140870`.) + New modules =========== From 3a7a22b5b0cf292ab9e7d046980085c227f7fdaa Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Fri, 4 Sep 2026 19:57:07 +0300 Subject: [PATCH 7/8] gh-156187: Fix the warning stacklevel inside a nested set operand (GH-156188) --- Lib/re/_parser.py | 2 +- Lib/test/test_re.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index aab9b59168015c..0751f941a7a3a6 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -606,7 +606,7 @@ def addmember(code): if allow_nested and sourcematch("["): # A nested set after an operator is the whole operand, used as-is (not # wrapped in a group); it cannot be combined with loose members. - compound = _parse_charset(source, state, nested + 1) + compound = _parse_charset(source, state, nested + 2) while True: this = sourceget() if this is None: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index ff106c1b341566..8e7611344b9150 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1537,6 +1537,10 @@ def test_set_operations(self): with warnings.catch_warnings(): warnings.simplefilter('error', FutureWarning) re.compile(r'[a-z--[aeiou]]') + # A reserved construct inside a nested operand warns against the caller. + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + re.compile(r'[\w--[[:digit:]]]') + self.assertEqual(w.filename, __file__) # Set union A||B == A or B (an explicit form of [AB]); flat operands # merge into one charset, otherwise the operations are alternated. @@ -1557,6 +1561,10 @@ def test_set_operations(self): self.assertEqual(re.findall(r'[\d~~1]', s), list('0123456789~')) self.assertEqual(w.filename, __file__) self.assertEqual(re.findall(r'[~~1]', s), list('1~')) + with self.assertWarnsRegex(FutureWarning, + 'Possible set symmetric difference ') as w: + re.compile(r'[\w--[\d~~1]]') + self.assertEqual(w.filename, __file__) def test_search_coverage(self): self.assertEqual(re.search(r"\s(b)", " b").group(1), "b") From e56f86fb6cc7cf14c255d88465e9d51f6e977b4c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:15:44 +0100 Subject: [PATCH 8/8] gh-143493: fix cleanup on errors in codegen_comprehension (#156374) --- Python/codegen.c | 80 ++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index e2ef40b4e30490..88e3aa8648fc58 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -5051,6 +5051,38 @@ pop_inlined_comprehension_state(compiler *c, location loc, return SUCCESS; } +static int +codegen_comprehension_init_container(compiler *c, location loc, int type, + int is_inlined, bool avoid_creation) +{ + int op; + switch (type) { + case COMP_LISTCOMP: + op = BUILD_LIST; + break; + case COMP_SETCOMP: + op = BUILD_SET; + break; + case COMP_DICTCOMP: + op = BUILD_MAP; + break; + default: + PyErr_Format(PyExc_SystemError, + "unknown comprehension type %d", type); + return ERROR; + } + + if (!avoid_creation) { + ADDOP_I(c, loc, op, 0); + if (is_inlined) { + ADDOP_I(c, loc, SWAP, 2); + } + } else { + ADDOP_I(c, loc, COPY, 1); + } + return SUCCESS; +} + static int codegen_comprehension(compiler *c, expr_ty e, int type, identifier name, asdl_comprehension_seq *generators, expr_ty elt, @@ -5089,19 +5121,22 @@ codegen_comprehension(compiler *c, expr_ty e, int type, if (type == COMP_GENEXP) { /* Insert GET_ITER before RETURN_GENERATOR. https://docs.python.org/3/reference/expressions.html#generator-expressions */ - RETURN_IF_ERROR( - _PyInstructionSequence_InsertInstruction( + if(_PyInstructionSequence_InsertInstruction( INSTR_SEQUENCE(c), 0, - RESUME, RESUME_AT_GEN_EXPR_START, NO_LOCATION)); - RETURN_IF_ERROR( - _PyInstructionSequence_InsertInstruction( + RESUME, RESUME_AT_GEN_EXPR_START, NO_LOCATION) < 0) { + goto error_in_scope; + } + if(_PyInstructionSequence_InsertInstruction( INSTR_SEQUENCE(c), 1, - LOAD_FAST, 0, LOC(outermost->iter))); - RETURN_IF_ERROR( - _PyInstructionSequence_InsertInstruction( + LOAD_FAST, 0, LOC(outermost->iter)) < 0) { + goto error_in_scope; + } + if(_PyInstructionSequence_InsertInstruction( INSTR_SEQUENCE(c), 2, outermost->is_async ? GET_AITER : GET_ITER, - 0, LOC(outermost->iter))); + 0, LOC(outermost->iter)) < 0) { + goto error_in_scope; + } iter_state = ITERATOR_ON_STACK; } else { @@ -5111,31 +5146,10 @@ codegen_comprehension(compiler *c, expr_ty e, int type, Py_CLEAR(entry); if (type != COMP_GENEXP) { - int op; - switch (type) { - case COMP_LISTCOMP: - op = BUILD_LIST; - break; - case COMP_SETCOMP: - op = BUILD_SET; - break; - case COMP_DICTCOMP: - op = BUILD_MAP; - break; - default: - PyErr_Format(PyExc_SystemError, - "unknown comprehension type %d", type); + if (codegen_comprehension_init_container( + c, loc, type, is_inlined, avoid_creation) < 0) { goto error_in_scope; } - - if (!avoid_creation) { - ADDOP_I(c, loc, op, 0); - if (is_inlined) { - ADDOP_I(c, loc, SWAP, 2); - } - } else { - ADDOP_I(c, loc, COPY, 1); - } } if (codegen_comprehension_generator(c, loc, generators, 0, 0, elt, val, type, iter_state, avoid_creation) < 0) { @@ -5150,7 +5164,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } if (type != COMP_GENEXP) { - ADDOP(c, LOC(e), RETURN_VALUE); + ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); } if (type == COMP_GENEXP) { if (codegen_wrap_in_stopiteration_handler(c) < 0) {