From 63f662663acd3751632e75fb03e7f1e52c4bb5e5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Aug 2026 19:48:23 +0200 Subject: [PATCH 1/7] gh-156101: Don't set the result in error in PyLong_AsInt32() (#156727) Don't set the result in error in PyLong_AsInt32(), PyLong_AsUInt32(), PyLong_AsInt64() and PyLong_AsUInt64(). Leave the result unchanged in this case. --- ...-08-31-19-14-15.gh-issue-156101.d4PSIF.rst | 4 +++ Modules/_testlimitedcapi/long.c | 12 ++++--- Objects/longobject.c | 36 ++++++++++--------- 3 files changed, 32 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-08-31-19-14-15.gh-issue-156101.d4PSIF.rst diff --git a/Misc/NEWS.d/next/C_API/2026-08-31-19-14-15.gh-issue-156101.d4PSIF.rst b/Misc/NEWS.d/next/C_API/2026-08-31-19-14-15.gh-issue-156101.d4PSIF.rst new file mode 100644 index 00000000000000..68822d6702e3bc --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-31-19-14-15.gh-issue-156101.d4PSIF.rst @@ -0,0 +1,4 @@ +Don't set the result in error in :c:func:`PyLong_AsInt32`, +:c:func:`PyLong_AsUInt32`, :c:func:`PyLong_AsInt64` and +:c:func:`PyLong_AsUInt64`. Leave the result unchanged in this case. Patch by +Victor Stinner. diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c index 7c6928dd234e32..1654f9210e1d0b 100644 --- a/Modules/_testlimitedcapi/long.c +++ b/Modules/_testlimitedcapi/long.c @@ -764,8 +764,9 @@ static PyObject * pylong_asint32(PyObject *module, PyObject *arg) { NULLABLE(arg); - int32_t value; + int32_t value = UNINITIALIZED_INT; if (PyLong_AsInt32(arg, &value) < 0) { + assert(value == UNINITIALIZED_INT); return NULL; } return PyLong_FromInt32(value); @@ -775,8 +776,9 @@ static PyObject * pylong_asuint32(PyObject *module, PyObject *arg) { NULLABLE(arg); - uint32_t value; + uint32_t value = UNINITIALIZED_INT; if (PyLong_AsUInt32(arg, &value) < 0) { + assert(value == UNINITIALIZED_INT); return NULL; } return PyLong_FromUInt32(value); @@ -787,8 +789,9 @@ static PyObject * pylong_asint64(PyObject *module, PyObject *arg) { NULLABLE(arg); - int64_t value; + int64_t value = UNINITIALIZED_INT; if (PyLong_AsInt64(arg, &value) < 0) { + assert(value == UNINITIALIZED_INT); return NULL; } return PyLong_FromInt64(value); @@ -798,8 +801,9 @@ static PyObject * pylong_asuint64(PyObject *module, PyObject *arg) { NULLABLE(arg); - uint64_t value; + uint64_t value = UNINITIALIZED_INT; if (PyLong_AsUInt64(arg, &value) < 0) { + assert(value == UNINITIALIZED_INT); return NULL; } return PyLong_FromUInt64(value); diff --git a/Objects/longobject.c b/Objects/longobject.c index 7a38ae8ea5a36f..6454565aebf6a1 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6800,58 +6800,62 @@ PyObject* PyLong_FromUInt64(uint64_t value) PYLONG_FROM_UINT(uint64_t, value); } -#define LONG_TO_INT(obj, value, type_name) \ +#define LONG_TO_INT(type, obj, result) \ do { \ + type value; \ int flags = (Py_ASNATIVEBYTES_NATIVE_ENDIAN \ | Py_ASNATIVEBYTES_ALLOW_INDEX); \ - Py_ssize_t bytes = PyLong_AsNativeBytes(obj, value, sizeof(*value), flags); \ + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, &value, sizeof(value), flags); \ if (bytes < 0) { \ return -1; \ } \ - if ((size_t)bytes > sizeof(*value)) { \ + if ((size_t)bytes > sizeof(value)) { \ PyErr_SetString(PyExc_OverflowError, \ - "Python int too large to convert to " type_name); \ + "Python int too large to convert to C " #type); \ return -1; \ } \ + *result = value; \ return 0; \ } while (0) -int PyLong_AsInt32(PyObject *obj, int32_t *value) +int PyLong_AsInt32(PyObject *obj, int32_t *result) { - LONG_TO_INT(obj, value, "C int32_t"); + LONG_TO_INT(int32_t, obj, result); } -int PyLong_AsInt64(PyObject *obj, int64_t *value) +int PyLong_AsInt64(PyObject *obj, int64_t *result) { - LONG_TO_INT(obj, value, "C int64_t"); + LONG_TO_INT(int64_t, obj, result); } -#define LONG_TO_UINT(obj, value, type_name) \ +#define LONG_TO_UINT(type, obj, result) \ do { \ + type value; \ int flags = (Py_ASNATIVEBYTES_NATIVE_ENDIAN \ | Py_ASNATIVEBYTES_UNSIGNED_BUFFER \ | Py_ASNATIVEBYTES_REJECT_NEGATIVE \ | Py_ASNATIVEBYTES_ALLOW_INDEX); \ - Py_ssize_t bytes = PyLong_AsNativeBytes(obj, value, sizeof(*value), flags); \ + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, &value, sizeof(value), flags); \ if (bytes < 0) { \ return -1; \ } \ - if ((size_t)bytes > sizeof(*value)) { \ + if ((size_t)bytes > sizeof(value)) { \ PyErr_SetString(PyExc_OverflowError, \ - "Python int too large to convert to " type_name); \ + "Python int too large to convert to C " #type); \ return -1; \ } \ + *result = value; \ return 0; \ } while (0) -int PyLong_AsUInt32(PyObject *obj, uint32_t *value) +int PyLong_AsUInt32(PyObject *obj, uint32_t *result) { - LONG_TO_UINT(obj, value, "C uint32_t"); + LONG_TO_UINT(uint32_t, obj, result); } -int PyLong_AsUInt64(PyObject *obj, uint64_t *value) +int PyLong_AsUInt64(PyObject *obj, uint64_t *result) { - LONG_TO_UINT(obj, value, "C uint64_t"); + LONG_TO_UINT(uint64_t, obj, result); } From 1855c6c40b02a5c8e75de2832dbb6cbd24351f1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Aug 2026 20:03:33 +0200 Subject: [PATCH 2/7] gh-155358: Use named attributes with urllib.parse module (#155364) urlparse(), replace: * parts[0] => parts.scheme * parts[1] => parts.netloc * parts[2] => parts.path urlsplit(), replace: * parts[0] => parts.scheme * parts[1] => parts.netloc * parts[2] => parts.path --- Lib/http/cookiejar.py | 2 +- Lib/test/ssl_servers.py | 2 +- Lib/test/support/__init__.py | 2 +- Lib/urllib/request.py | 12 ++++++------ Lib/urllib/robotparser.py | 4 +++- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 144db91a0f0317..5c5dc78b064885 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -627,7 +627,7 @@ def request_host(request): """ url = request.get_full_url() - host = urllib.parse.urlparse(url)[1] + host = urllib.parse.urlparse(url).netloc if host == "": host = request.get_header("Host", "") diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py index 15b071e04dda1f..e3416a822f6525 100644 --- a/Lib/test/ssl_servers.py +++ b/Lib/test/ssl_servers.py @@ -61,7 +61,7 @@ def translate_path(self, path): """ # abandon query parameters - path = urllib.parse.urlparse(path)[2] + path = urllib.parse.urlparse(path).path path = os.path.normpath(urllib.parse.unquote(path)) words = path.split('/') words = filter(None, words) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index be71575a6ea06a..1c28af08988d9f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -868,7 +868,7 @@ def open_urlresource(url, *args, **kw): check = kw.pop('check', None) - filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL! + filename = urllib.parse.urlparse(url).path.split('/')[-1] # '/': it's URL! fn = os.path.join(TEST_DATA_DIR, filename) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 9fa92659a255ed..59ed4a7140d59c 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -274,7 +274,7 @@ def request_host(request): """ url = request.full_url - host = urlparse(url)[1] + host = urlparse(url).netloc if host == "": host = request.get_header("Host", "") @@ -833,11 +833,11 @@ def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" # note HTTP URLs do not have a userinfo component parts = urlsplit(uri) - if parts[1]: + if parts.netloc: # URI - scheme = parts[0] - authority = parts[1] - path = parts[2] or '/' + scheme = parts.scheme + authority = parts.netloc + path = parts.path or '/' else: # host or host:port scheme = None @@ -1222,7 +1222,7 @@ class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler): handler_order = 490 # before Basic auth def http_error_401(self, req, fp, code, msg, headers): - host = urlparse(req.full_url)[1] + host = urlparse(req.full_url).netloc retry = self.http_error_auth_reqed('www-authenticate', host, req, headers) self.reset_retry_count() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 8d0311d96f5e0b..985333c7100438 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -62,7 +62,9 @@ def set_url(self, url): if isinstance(url, urllib.request.Request): url = url.full_url - self.host, self.path = urllib.parse.urlsplit(url)[1:3] + parts = urllib.parse.urlsplit(url) + self.host = parts.netloc + self.path = parts.path def read(self): """Reads the robots.txt URL and feeds it to the parser.""" From 287b7cffb79d443614be51b48eaf085d663aeecd Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 31 Aug 2026 19:16:07 +0100 Subject: [PATCH 3/7] gh-156723: Update bundled libexpat to version 2.8.4 (#156724) --- ...08-31-16-47-33.gh-issue-156723.KxCF5N.rst} | 3 +- Misc/sbom.spdx.json | 24 +- Modules/expat/expat.h | 2 +- Modules/expat/internal.h | 1 + Modules/expat/refresh.sh | 6 +- Modules/expat/xmlparse.c | 253 +++++++++++++----- Modules/expat/xmltok.h | 4 +- 7 files changed, 209 insertions(+), 84 deletions(-) rename Misc/NEWS.d/next/Security/{2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst => 2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst} (59%) diff --git a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst similarity index 59% rename from Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst rename to Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst index 439366c8633e82..3dda6055f307d9 100644 --- a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst +++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst @@ -1,2 +1 @@ -Update bundled `libexpat `_ to version 2.8.3 -for the fix to :cve:`2026-72522`. +Update bundled `libexpat `_ to version 2.8.4. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 3e4feba10c8559..8708cef4e2ab26 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7baecf6e04769cfb0c5ce2a6e3241e3a0bb8c9e9" + "checksumValue": "12dffaa4a67cbe308643dbec7ffc1b4fd38abbde" }, { "algorithm": "SHA256", - "checksumValue": "d3f19ed52dc975741ecc5a0fc553f910a241d60c76fa4621356d0cdb0490ca28" + "checksumValue": "0e912e25375e213b6e4ff90d554e0a0e037e6f0c20dfa734d366e5bdff289f20" } ], "fileName": "Modules/expat/expat.h" @@ -104,11 +104,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "476a11d9872f8f38844e398c5486ad183ffe2dcf" + "checksumValue": "4afd563c90edd6b4aa5abedcd3df5df023668d26" }, { "algorithm": "SHA256", - "checksumValue": "89f661fa3fa5f7892d83a13ecd685a56aace3fe740abce88a863031114ee2cef" + "checksumValue": "beb7211c800d827743bd3d6ddb86538302d6c51180be6d3b61a1c315e061762d" } ], "fileName": "Modules/expat/internal.h" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0939e3fe0ebb21a5b8ed9d9fdd33cde75ee5658a" + "checksumValue": "b9e4628f37353a7eec8a98c26ebb88d7e2d48971" }, { "algorithm": "SHA256", - "checksumValue": "da48375e85bdc2f97da4445169aafc0b363f150a1a8275dd417e6d84cfc3e443" + "checksumValue": "9afa5cb812283750f1970e230ba392201bac46240abf51ab65e5090e93ca34a6" } ], "fileName": "Modules/expat/xmlparse.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8e4bf167669dddff38269486f33eccb0fde0c7ca" + "checksumValue": "e9a5972f664c1c530443ddd8b7900e406e3ca02a" }, { "algorithm": "SHA256", - "checksumValue": "20013b75027e04e324452a002100076e30ec20e0f28b318f392317f99a4c4115" + "checksumValue": "41a6cef659ef1da9ee732304332c4134afe4b63571442de77eaf9918abf9e5df" } ], "fileName": "Modules/expat/xmltok.h" @@ -1044,14 +1044,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" + "checksumValue": "b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_4/expat-2.8.4.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.4:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1059,7 +1059,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.3" + "versionInfo": "2.8.4" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index dbebd985a652ac..b296be9dbad2dc 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1096,7 +1096,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 3 +# define XML_MICRO_VERSION 4 # ifdef __cplusplus } diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 7e67d2e378c524..6311028e94b8f5 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -33,6 +33,7 @@ Copyright (c) 2019 David Loffredo Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 1499e92112fb95..ef06e87122aa26 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_3" -expected_libexpat_version="2.8.3" -expected_libexpat_sha256="22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" +expected_libexpat_tag="R_2_8_4" +expected_libexpat_version="2.8.4" +expected_libexpat_sha256="b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 4fa61bca8c1629..9a05da21d5a7fd 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* ee5f82c3ffd57c5224394ba46f348dbce466d34d6c925a527ae46b1cfe6adf1d (2.8.3+) +/* 13c4e8da8fccffb0e8e599684e0d447ad14c1bb0b48792cf5dd77d8712301871 (2.8.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -51,6 +51,9 @@ Copyright (c) 2026 Kartik Kenchi Copyright (c) 2026 Haris Hussain Copyright (c) 2026 Evgeny Kotkov + Copyright (c) 2026 Darren Carreras + Copyright (c) 2026 Alberto Maschietto + Copyright (c) 2026 Zeyou Liu Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -330,7 +333,7 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; + bool open; XML_Bool hasMore; /* true if entity has not been completely processed */ /* An entity can be open while being already completely processed (hasMore == XML_FALSE). The reason is the delayed closing of entities until their inner @@ -381,6 +384,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -394,7 +413,7 @@ typedef struct { size_t nDefaultAtts; size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; - HASH_TABLE defaultAttsNames; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -579,6 +598,8 @@ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); +static NAMED *lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, + size_t nameLen, size_t createSize); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); @@ -755,6 +776,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -1177,6 +1200,25 @@ isCalledFromInsideHandler(XML_Parser parser) { return parser->m_handlerCallDepth > 0; } +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1524,6 +1566,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; @@ -1604,7 +1647,7 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); @@ -1915,7 +1958,7 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); FREE(parser, parser); } @@ -2739,7 +2782,7 @@ XML_GetCurrentLineNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -2756,7 +2799,7 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -3410,9 +3453,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; beforeHandler(parser); @@ -3837,8 +3880,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; - if (! elementType->defaultAttsNames.parser) - hashTableInit(&(elementType->defaultAttsNames), parser); + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -3951,11 +3994,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - for (size_t j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } @@ -4046,8 +4092,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4946,25 +4992,34 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { const int status = parser->m_unknownEncodingHandler( parser->m_unknownEncodingHandlerData, encodingName, &info); afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } @@ -6092,7 +6147,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6101,11 +6156,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -6429,7 +6484,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; + entity->open = true; entity->hasMore = XML_TRUE; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); @@ -6520,7 +6575,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // to false. This means we can directly remove the head of // m_openInternalEntities assert(parser->m_openInternalEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ @@ -6598,7 +6653,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // with hasMore set to false. This means we can directly remove the head // of m_openAttributeEntities assert(parser->m_openAttributeEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ @@ -6894,7 +6949,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (entity->systemId) { if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6903,12 +6958,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else @@ -7058,7 +7113,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // with hasMore set to false. This means we can directly remove the head // of m_openValueEntities assert(parser->m_openValueEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ @@ -7239,7 +7294,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttForName), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) @@ -7275,11 +7330,24 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound - = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); - if (! nameAddedOrFound) + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) return 0; + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -7480,7 +7548,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -7597,7 +7665,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); @@ -7639,7 +7707,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); @@ -7732,8 +7800,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; - if (! newE->defaultAttsNames.parser) - hashTableInit(&(newE->defaultAttsNames), parser); + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. */ @@ -7766,11 +7834,22 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), - attributeName, sizeof(NAMED)); - if (! nameAddedOrFound) { + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { return 0; } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7867,19 +7946,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { +keyeq(KEY s1, size_t s1len, KEY s2) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T - return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; # else - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; # endif #else - return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; #endif } @@ -7897,18 +7980,38 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { } static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7924,14 +8027,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7964,7 +8067,8 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7987,15 +8091,36 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } + assert(createSize >= sizeof(NAMED)); table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +static NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + static void FASTCALL hashTableClear(HASH_TABLE *table) { size_t i; @@ -8535,8 +8660,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; - if (! ret->defaultAttsNames.parser) - hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index bd868b87a407d6..76be2c7c5ca1ba 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -169,8 +169,8 @@ typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding { From 8e92bf54e7ae55497a1b193b875d07e10e52d0e1 Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:24:03 +0300 Subject: [PATCH 4/7] gh-155418: Fix TaskGroup hang when a task cancels it before suspending (#155421) --- Lib/asyncio/taskgroups.py | 3 +++ Lib/test/test_asyncio/test_taskgroups.py | 11 +++++++++++ .../2026-08-09-14-27-23.gh-issue-155418.kCXUIG.rst | 2 ++ 3 files changed, 16 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-09-14-27-23.gh-issue-155418.kCXUIG.rst diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 955e8e677eac1a..2debb4d5849e68 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -239,6 +239,9 @@ def create_task(self, coro, **kwargs): # the current task too early. gh-128550, gh-128588 self._tasks.add(task) task.add_done_callback(self._on_task_done) + # gh-155418: an eager task can cancel the group before joining _tasks + if self._aborting and not task.done(): + task.cancel() try: return task finally: diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 983e1a7dc53e6f..1515672393816b 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1187,6 +1187,17 @@ async def test_taskgroup_cancel_before_create_task(self): with self.assertRaises(RuntimeError): tg.create_task(asyncio.sleep(1)) + async def test_taskgroup_cancel_from_child_before_first_suspension(self): + # gh-155418: an eager task can cancel the group before joining _tasks + async def child(tg): + tg.cancel() + await asyncio.sleep(10) + self.fail("the child was not cancelled") + + async with asyncio.TaskGroup() as tg: + task = tg.create_task(child(tg)) + self.assertTrue(task.cancelled()) + async def test_taskgroup_cancel_keeps_outer_cancellation(self): # gh-155433: any cancellation from outside the group must propagate. async def child(): diff --git a/Misc/NEWS.d/next/Library/2026-08-09-14-27-23.gh-issue-155418.kCXUIG.rst b/Misc/NEWS.d/next/Library/2026-08-09-14-27-23.gh-issue-155418.kCXUIG.rst new file mode 100644 index 00000000000000..7fe30ddb1ce8b7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-09-14-27-23.gh-issue-155418.kCXUIG.rst @@ -0,0 +1,2 @@ +Fix :class:`asyncio.TaskGroup` hang when a task created by +:func:`asyncio.eager_task_factory` cancels the group before suspending. From 486b000c6c19c555f03b481f735f4dec498f0f67 Mon Sep 17 00:00:00 2001 From: Som Samantray <92726151+SomSamantray@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:09:49 +0530 Subject: [PATCH 5/7] gh-121291: Respect mixin bitwise-operator overrides on Flag subclasses (GH-155862) * gh-121291: respect mixin-defined bitwise operators on Flag subclasses EnumMeta.__new__ unconditionally installed Flag's __or__/__and__/__xor__/ __ror__/__rand__/__rxor__/__invert__ onto every Flag subclass, silently discarding a mixin base's own override of these dunders -- even though the class's MRO should have resolved to the mixin's method; this fixes that. --- Lib/enum.py | 8 ++- Lib/test/test_enum.py | 53 +++++++++++++++++++ ...-08-15-18-16-16.gh-issue-121291.WljPkh.rst | 4 ++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst diff --git a/Lib/enum.py b/Lib/enum.py index 7aff36c94ce1dc..076aa18a02fd20 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -624,9 +624,13 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k '__invert__' ): if name not in classdict: + # check for mixin overrides before replacing enum_method = getattr(Flag, name) - setattr(enum_class, name, enum_method) - classdict[name] = enum_method + found_method = getattr(enum_class, name) + data_type_method = getattr(member_type, name, None) + if found_method in (enum_method, data_type_method): + setattr(enum_class, name, enum_method) + classdict[name] = enum_method # # replace any other __new__ with our own (as long as Enum is not None, # anyway) -- again, this is to support pickle diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b05eab43bd9eff..447f847f33da93 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4080,6 +4080,39 @@ class NeverEnum(WhereEnum): self.assertFalse(NeverEnum.__dict__.get('_test1', False)) self.assertFalse(NeverEnum.__dict__.get('_test2', False)) + def test_mixin_operator_override(self): + # a mixin's own bitwise-operator overrides must not be clobbered + # by Flag's default __or__/__and__/__xor__/__invert__ -- gh-121291 + class OperatorMixin: + def __or__(self, other): + return 'mixin-or' + def __ror__(self, other): + return 'mixin-ror' + def __invert__(self): + return 'mixin-invert' + class MixedFlag(OperatorMixin, Flag): + A = 1 + B = 2 + self.assertIs(MixedFlag.__or__, OperatorMixin.__or__) + self.assertIs(MixedFlag.__ror__, OperatorMixin.__ror__) + self.assertIs(MixedFlag.__invert__, OperatorMixin.__invert__) + self.assertEqual(MixedFlag.A | MixedFlag.B, 'mixin-or') + self.assertEqual(1 | MixedFlag.A, 'mixin-ror') + self.assertEqual(~MixedFlag.A, 'mixin-invert') + # dunders the mixin didn't override still get Flag's own + self.assertIs(MixedFlag.__and__, Flag.__and__) + self.assertIs(MixedFlag.__xor__, Flag.__xor__) + self.assertIs(MixedFlag.__rand__, Flag.__rand__) + self.assertIs(MixedFlag.__rxor__, Flag.__rxor__) + self.assertEqual(MixedFlag.A & MixedFlag.B, MixedFlag(0)) + # + # a plain (non-mixin) Flag subclass is unaffected + class PlainFlag(Flag): + A = 1 + B = 2 + self.assertIs(PlainFlag.__or__, Flag.__or__) + self.assertEqual(PlainFlag.A | PlainFlag.B, PlainFlag(3)) + class OldTestIntFlag(unittest.TestCase): """Tests of the IntFlags.""" @@ -4564,6 +4597,26 @@ def cycle_enum(): 'at least one thread failed while creating composite members') self.assertEqual(256, len(seen), 'too many composite members created') + def test_mixin_operator_override(self): + # IntFlag's own mixed-in `int` also defines these operators, so the + # fix for gh-121291 must still override `int`'s raw operators with + # Flag's (returning IntFlag instances, not plain ints), while still + # respecting a genuine, separate mixin's override. + Color = self.Color + combined = Color.RED | Color.BLUE + self.assertIs(type(combined), Color) + self.assertEqual(combined, Color.PURPLE) + self.assertEqual(repr(combined), '') + # + class OperatorMixin: + def __or__(self, other): + return 'mixin-or' + class MixedIntFlag(OperatorMixin, IntFlag): + A = 1 + B = 2 + self.assertIs(MixedIntFlag.__or__, OperatorMixin.__or__) + self.assertEqual(MixedIntFlag.A | MixedIntFlag.B, 'mixin-or') + class TestEmptyAndNonLatinStrings(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst b/Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst new file mode 100644 index 00000000000000..e6d9f4b69ec9e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst @@ -0,0 +1,4 @@ +:class:`enum.Flag` (and :class:`enum.IntFlag`) subclasses no longer have +a mixin base's own ``__or__``, ``__and__``, ``__xor__``, ``__ror__``, +``__rand__``, ``__rxor__``, or ``__invert__`` override silently replaced +by :class:`~enum.Flag`'s default implementation. From c83013c92dfdc77b87a523b736b76d5abb8ede2a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 1 Sep 2026 00:27:24 +0300 Subject: [PATCH 6/7] gh-99064: Ignore the encoding declaration when parsing decoded text (GH-156734) ElementTree.parse() with a text file mis-decoded the text in the C implementation: _parse_whole() encoded it as UTF-8, but left expat to honor the encoding declared in the document. It now overrides the encoding, as XMLParser.feed() already does for str data. --- Lib/test/test_xml_etree.py | 28 +++++++++++++++++++ ...6-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst | 5 ++++ Modules/_elementtree.c | 7 +++++ 3 files changed, 40 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 2af2d1fd64520b..fb35bb6a5f442f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1068,6 +1068,34 @@ def bxml(encoding, body=''): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) + def test_parse_text_source(self): + # gh-99064: The encoding declared in the document does not apply + # to a source which is already decoded. + def check(encoding, body): + xml = (f"" + f"{body}") + with self.subTest(encoding=encoding): + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, + body) + # the same with an explicitly created parser + self.assertEqual( + ET.parse(io.StringIO(xml), ET.XMLParser()).getroot().text, + body) + check("ascii", 'a') + check("iso-8859-1", '\xbd') + check("iso-8859-15", '\u20ac') + check("cp437", '\u221a') + check("utf-8", '\u4e2d') + # not ASCII compatible, unsupported for a bytes source + check("utf-16", '\u4e2d') + check("utf-32", '\u4e2d') + + def test_parse_text_source_multiple_chunks(self): + # the encoding is overridden before the first chunk is parsed + body = '\xe4' * 100_000 + xml = "%s" % body + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) + @support.subTests('sample,exception', [ (b' \xa1', UnicodeDecodeError), # crashed (b' \xa1state; + int first = 1; for (;;) { buffer = PyObject_CallFunction(reader, "i", 64*1024); @@ -4100,6 +4101,11 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, Py_DECREF(buffer); break; } + if (first) { + /* The text is already decoded, the encoding declared in the + document does not apply to it. Return code ignored. */ + (void)EXPAT(st, SetEncoding)(self->parser, "utf-8"); + } temp = PyUnicode_AsEncodedString(buffer, "utf-8", "surrogatepass"); Py_DECREF(buffer); if (!temp) { @@ -4123,6 +4129,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, res = expat_parse( st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer), 0); + first = 0; Py_DECREF(buffer); From b38e073f1be8fe40af991c044a791cefff098f0d Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 31 Aug 2026 14:54:35 -0700 Subject: [PATCH 7/7] gh-155981: Store refleak deltas in arrays (gh-55982) Store per-run deltas in array objects rather than lists of pooled integers. Large, unique deltas could otherwise grow int_pool and make the refleak checker report its own retained integers as reference leaks. --- Lib/test/libregrtest/refleak.py | 34 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index e7da17e500ead9..ffb8438d1b0278 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,6 +1,7 @@ import os import sys import warnings +from array import array from inspect import isabstract from typing import Any import linecache @@ -100,24 +101,20 @@ def runtest_refleak(test_name, test_func, for obj in ByteString.__subclasses__() + [ByteString]: # type: ignore[attr-defined] abcs[obj] = _get_dump(obj)[0] - # bpo-31217: Integer pool to get a single integer object for the same - # value. The pool is used to prevent false alarm when checking for memory - # block leaks. Fill the pool with values in -1000..1000 which are the most - # common (reference, memory block, file descriptor) differences. - int_pool = {value: value for value in range(-1000, 1000)} - def get_pooled_int(value): - return int_pool.setdefault(value, value) - warmups = hunt_refleak.warmups runs = hunt_refleak.runs filename = hunt_refleak.filename repcount = warmups + runs - # Pre-allocate to ensure that the loop doesn't allocate anything new + # Pre-allocate to ensure that the loop doesn't allocate anything new. + # Store the deltas as raw values in arrays rather than as int objects in + # lists: each unique delta stored as an object would live until the end of + # the loop and show up in the following repetition's reference and memory + # block deltas (gh-75400, gh-155981). rep_range = list(range(repcount)) - rc_deltas = [0] * repcount - alloc_deltas = [0] * repcount - fd_deltas = [0] * repcount + rc_deltas = array('q', [0]) * repcount + alloc_deltas = array('q', [0]) * repcount + fd_deltas = array('q', [0]) * repcount getallocatedblocks = sys.getallocatedblocks gettotalrefcount = sys.gettotalrefcount getunicodeinternedsize = sys.getunicodeinternedsize @@ -161,12 +158,11 @@ def get_pooled_int(value): rc_after = gettotalrefcount() fd_after = fd_count() - rc_deltas[i] = get_pooled_int(rc_after - rc_before) - alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) - fd_deltas[i] = get_pooled_int(fd_after - fd_before) + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + fd_deltas[i] = fd_after - fd_before if not quiet: - # use max, not sum, so total_leaks is one of the pooled ints total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) if total_leaks <= 0: symbol = '.' @@ -212,13 +208,13 @@ def check_fd_deltas(deltas): return any(deltas) failed = False - for deltas, item_name, checker in [ + for raw_deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), (alloc_deltas, 'memory blocks', check_rc_deltas), (fd_deltas, 'file descriptors', check_fd_deltas) ]: - # ignore warmup runs - deltas = deltas[warmups:] + # ignore warmup runs; convert to a list for reporting + deltas = list(raw_deltas[warmups:]) failing = checker(deltas) suspicious = any(deltas) if failing or suspicious: