From fa0ec86ab682a561e2dc6f50127f2a78f76ce311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Tue, 18 Aug 2026 02:13:33 +0800 Subject: [PATCH 1/4] gh-153539: Fix use-after-free in TextIOWrapper.tell() with a reentrant decoder (GH-153540) TextIOWrapper.tell() used a borrowed next_input from the snapshot across the decoder's getstate/decode/setstate calls, so a decoder that reenters seek() from getstate could free it and leave tell() reading freed memory. Own the reference across those calls, matching the sibling textiowrapper_read_chunk. --- Lib/test/test_io/test_textio.py | 43 +++++++++++++++++++ ...-07-11-12-21-12.gh-issue-153539.gBs2T5.rst | 3 ++ Modules/_io/textio.c | 8 +++- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-11-12-21-12.gh-issue-153539.gBs2T5.rst diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index 82096ab09873955..07f6b1415d0dbda 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -1610,6 +1610,49 @@ def make_text(buffer): wrapper.write('x') self.assertRaisesRegex(ValueError, "detached", wrapper.read) + def test_reentrant_seek_during_tell(self): + # gh-153539: reading short of _CHUNK_SIZE leaves residual bytes in the + # snapshot, so tell() re-decodes and calls the decoder's getstate(); a + # reentrant seek() there must not free the snapshot tell() still uses. + # C-only: _pyio binds next_input as a strong local and cannot crash. + wrapper = None + armed = False + + class ReentrantDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return bytes(input).decode("latin-1") + def getstate(self): + nonlocal armed + if wrapper is not None and armed: + armed = False + wrapper.seek(0) + return (b"", 0) + def setstate(self, state): + pass + + def search(name): + if name != "reentrant_tell_test": + return None + return codecs.CodecInfo( + name=name, + encode=lambda s, e='strict': (s.encode("latin-1"), len(s)), + decode=lambda b, e='strict': (bytes(b).decode("latin-1"), len(b)), + incrementaldecoder=ReentrantDecoder) + + codecs.register(search) + self.addCleanup(codecs.unregister, search) + raw = self.BytesIO(b"abcdefghijklmnop" * 8) + wrapper = self.TextIOWrapper(self.BufferedReader(raw), + encoding="reentrant_tell_test", newline="") + wrapper._CHUNK_SIZE = 8 + wrapper.read(5) + armed = True + self.assertIsInstance(wrapper.tell(), int) + # tell() at the snapshot boundary takes the early return that owns and + # must release next_input; exercise it too (leak-checked under -R). + wrapper.seek(0) + self.assertIsInstance(wrapper.tell(), int) + class PyTextIOWrapperTest(TextIOWrapperTest, PyTestCase): shutdown_error = "LookupError: unknown encoding: ascii" diff --git a/Misc/NEWS.d/next/Library/2026-07-11-12-21-12.gh-issue-153539.gBs2T5.rst b/Misc/NEWS.d/next/Library/2026-07-11-12-21-12.gh-issue-153539.gBs2T5.rst new file mode 100644 index 000000000000000..241d67949680925 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-11-12-21-12.gh-issue-153539.gBs2T5.rst @@ -0,0 +1,3 @@ +Fix a crash in the C implementation of :meth:`io.TextIOWrapper.tell` when the +decoder's ``getstate`` method triggers a reentrant seek, or when another thread +seeks the same stream concurrently. Patch by tonghuaroot. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index ea8ed2713d8a146..5b3d379e4e75543 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2823,7 +2823,7 @@ _io_TextIOWrapper_tell_impl(textio *self) PyObject *res; PyObject *posobj = NULL; cookie_type cookie = {0,0,0,0,0}; - PyObject *next_input; + PyObject *next_input = NULL; Py_ssize_t chars_to_skip, chars_decoded; Py_ssize_t skip_bytes, skip_back; PyObject *saved_state = NULL; @@ -2875,11 +2875,15 @@ _io_TextIOWrapper_tell_impl(textio *self) assert (PyBytes_Check(next_input)); + /* Own next_input: a reentrant or concurrent seek can drop the snapshot. */ + Py_INCREF(next_input); + cookie.start_pos -= PyBytes_GET_SIZE(next_input); /* How many decoded characters have been used up since the snapshot? */ if (self->decoded_chars_used == 0) { /* We haven't moved from the snapshot point. */ + Py_DECREF(next_input); return textiowrapper_build_cookie(&cookie); } @@ -3020,6 +3024,7 @@ _io_TextIOWrapper_tell_impl(textio *self) } finally: + Py_XDECREF(next_input); res = PyObject_CallMethodOneArg( self->decoder, &_Py_ID(setstate), saved_state); Py_DECREF(saved_state); @@ -3032,6 +3037,7 @@ _io_TextIOWrapper_tell_impl(textio *self) return textiowrapper_build_cookie(&cookie); fail: + Py_XDECREF(next_input); if (saved_state) { PyObject *exc = PyErr_GetRaisedException(); res = PyObject_CallMethodOneArg( From bf2c0539334877ce4bfb2b3c6a36248a0ef52d72 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 17 Aug 2026 21:58:29 +0300 Subject: [PATCH 2/4] gh-67217: Test both warnings implementations in environment variable tests (GH-155234) The code executed in the subprocess now disables the C implementation when the Python implementation is tested. Tests which only check sys.warnoptions do not depend on the implementation. They are moved to a separate test class and use the runInSubprocess() decorator. Co-authored-by: Claude Opus 5 (1M context) --- Lib/test/test_warnings/__init__.py | 103 ++++++++++++++++------------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index cde6076dd2f44b7..b86a8838c7a2457 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -13,6 +13,7 @@ import unittest from test import support from test.support import import_helper +from test.support import isolation from test.support import os_helper from test.support import warnings_helper from test.support import force_not_colorized @@ -1518,49 +1519,74 @@ class PyCatchWarningTests(CatchWarningTests, unittest.TestCase): module = py_warnings -class EnvironmentVariableTests(BaseTest): +_NONASCII_WARNOPTION = 'ignore:DeprecationWarning' + os_helper.FS_NONASCII + + +class WarnOptionsTests(unittest.TestCase): + """Tests of the -W option and the PYTHONWARNINGS environment variable. + + sys.warnoptions is set by the interpreter, so these tests do not depend + on the used implementation of the warnings module. + """ + @isolation.runInSubprocess( + env={'PYTHONWARNINGS': 'ignore::DeprecationWarning', + 'PYTHONDEVMODE': ''}) def test_single_warning(self): - rc, stdout, stderr = assert_python_ok("-c", - "import sys; sys.stdout.write(str(sys.warnoptions))", - PYTHONWARNINGS="ignore::DeprecationWarning", - PYTHONDEVMODE="") - self.assertEqual(stdout, b"['ignore::DeprecationWarning']") + self.assertEqual(sys.warnoptions, ['ignore::DeprecationWarning']) + @isolation.runInSubprocess( + env={'PYTHONWARNINGS': 'ignore::DeprecationWarning,' + 'ignore::UnicodeWarning', + 'PYTHONDEVMODE': ''}) def test_comma_separated_warnings(self): - rc, stdout, stderr = assert_python_ok("-c", - "import sys; sys.stdout.write(str(sys.warnoptions))", - PYTHONWARNINGS="ignore::DeprecationWarning,ignore::UnicodeWarning", - PYTHONDEVMODE="") - self.assertEqual(stdout, - b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + self.assertEqual(sys.warnoptions, ['ignore::DeprecationWarning', + 'ignore::UnicodeWarning']) - @force_not_colorized + @isolation.runInSubprocess( + options=['-Wignore::UnicodeWarning'], + env={'PYTHONWARNINGS': 'ignore::DeprecationWarning', + 'PYTHONDEVMODE': ''}) def test_envvar_and_command_line(self): - rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", - "import sys; sys.stdout.write(str(sys.warnoptions))", - PYTHONWARNINGS="ignore::DeprecationWarning", - PYTHONDEVMODE="") - self.assertEqual(stdout, - b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + self.assertEqual(sys.warnoptions, ['ignore::DeprecationWarning', + 'ignore::UnicodeWarning']) + + @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', + 'requires non-ascii filesystemencoding') + @isolation.runInSubprocess( + env={'PYTHONWARNINGS': _NONASCII_WARNOPTION, + 'PYTHONDEVMODE': ''}) + def test_nonascii(self): + self.assertEqual(sys.warnoptions, [_NONASCII_WARNOPTION]) + + +class EnvironmentVariableTests(BaseTest): + + def prepare_code(self, code): + """Make the subprocess use the tested implementation.""" + if self.module is py_warnings: + # Disable the warnings acceleration module in the subprocess. + code = ("import sys; sys.modules.pop('warnings', None); " + "sys.modules['_warnings'] = None; ") + code + return code @force_not_colorized def test_conflicting_envvar_and_command_line(self): - rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", + code = self.prepare_code( "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); " - "warnings.warn('Message', DeprecationWarning)", + "warnings.warn('Message', DeprecationWarning)") + rc, stdout, stderr = assert_python_failure( + "-Werror::DeprecationWarning", "-c", code, PYTHONWARNINGS="default::DeprecationWarning", PYTHONDEVMODE="") self.assertEqual(stdout, b"['default::DeprecationWarning', 'error::DeprecationWarning']") - self.assertEqual(stderr.splitlines(), - [b"Traceback (most recent call last):", - b" File \"\", line 1, in ", - b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' - b"arn('Message', DeprecationWarning)", - b' ~~~~~~~~~~' - b'~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', - b"DeprecationWarning: Message"]) + # The traceback of the Python implementation contains additional + # frames, so only the ends of the traceback are checked. + lines = stderr.splitlines() + self.assertEqual(lines[0], b"Traceback (most recent call last):") + self.assertEqual(lines[1], b' File "", line 1, in ') + self.assertEqual(lines[-1], b"DeprecationWarning: Message") def test_default_filter_configuration(self): pure_python_api = self.module is py_warnings @@ -1580,12 +1606,8 @@ def test_default_filter_configuration(self): ] expected_output = [str(f).encode() for f in expected_default_filters] - if pure_python_api: - # Disable the warnings acceleration module in the subprocess - code = "import sys; sys.modules.pop('warnings', None); sys.modules['_warnings'] = None; " - else: - code = "" - code += "import warnings; [print(f) for f in warnings._get_filters()]" + code = self.prepare_code( + "import warnings; [print(f) for f in warnings._get_filters()]") rc, stdout, stderr = assert_python_ok("-c", code, __isolated=True) stdout_lines = [line.strip() for line in stdout.splitlines()] @@ -1593,17 +1615,6 @@ def test_default_filter_configuration(self): self.assertEqual(stdout_lines, expected_output) - @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', - 'requires non-ascii filesystemencoding') - def test_nonascii(self): - PYTHONWARNINGS="ignore:DeprecationWarning" + os_helper.FS_NONASCII - rc, stdout, stderr = assert_python_ok("-c", - "import sys; sys.stdout.write(str(sys.warnoptions))", - PYTHONIOENCODING="utf-8", - PYTHONWARNINGS=PYTHONWARNINGS, - PYTHONDEVMODE="") - self.assertEqual(stdout, str([PYTHONWARNINGS]).encode()) - class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): module = c_warnings From b94b9c8886a987a324a677b5fda5bef27f15cb14 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Mon, 17 Aug 2026 22:20:07 +0300 Subject: [PATCH 3/4] gh-153005: Use a monotonic clock for concurrent.interpreters Queue timeouts (GH-154156) Queue.get() and Queue.put() computed their timeout deadline from time.time(), the wall clock. If the system clock was stepped (NTP, a manual change) while a call was blocked, the timeout could over- or under-wait. queue.Queue uses time.monotonic() for the same reason. Compute the deadline and check it against time.monotonic() instead. --- Lib/concurrent/interpreters/_queues.py | 8 ++++---- Lib/test/test_interpreters/test_queues.py | 15 +++++++++++++++ ...2026-07-19-17-45-00.gh-issue-153005.F6WH92.rst | 4 ++++ 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-17-45-00.gh-issue-153005.F6WH92.rst diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index 5f3ee0934de59d6..fc4ee595f3aa995 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -220,12 +220,12 @@ def put(self, obj, block=True, timeout=None, *, timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout + end = time.monotonic() + timeout while True: try: _queues.put(self._id, obj, unboundop) except QueueFull: - if timeout is not None and time.time() >= end: + if timeout is not None and time.monotonic() >= end: raise # re-raise time.sleep(_delay) else: @@ -255,12 +255,12 @@ def get(self, block=True, timeout=None, *, timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout + end = time.monotonic() + timeout while True: try: obj, unboundop = _queues.get(self._id) except QueueEmpty: - if timeout is not None and time.time() >= end: + if timeout is not None and time.monotonic() >= end: raise # re-raise time.sleep(_delay) else: diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 77334aea3836b98..baa772b3b367952 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -2,7 +2,9 @@ import pickle import threading from textwrap import dedent +import time import unittest +from unittest import mock from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. @@ -354,6 +356,19 @@ def test_get_timeout(self): with self.assertRaises(queues.QueueEmpty): queue.get(HUGE_TIMEOUT, 0.1) + def test_timeout_uses_monotonic_clock(self): + # gh-153005: the deadline must be computed from the monotonic clock, + # since the wall clock can be adjusted while the call is blocked. + queue = queues.create(1) + with mock.patch.object(queues, 'time', wraps=time) as fake_time: + with self.assertRaises(queues.QueueEmpty): + queue.get(timeout=0) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0) + fake_time.monotonic.assert_called() + fake_time.time.assert_not_called() + def test_get_nowait(self): queue = queues.create() with self.assertRaises(queues.QueueEmpty): diff --git a/Misc/NEWS.d/next/Library/2026-07-19-17-45-00.gh-issue-153005.F6WH92.rst b/Misc/NEWS.d/next/Library/2026-07-19-17-45-00.gh-issue-153005.F6WH92.rst new file mode 100644 index 000000000000000..7f3d10b3072ef94 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-17-45-00.gh-issue-153005.F6WH92.rst @@ -0,0 +1,4 @@ +:meth:`!concurrent.interpreters.Queue.get` and +:meth:`!concurrent.interpreters.Queue.put` now compute their ``timeout`` +deadline from :func:`time.monotonic` instead of the wall clock, so adjusting +the system clock during the call no longer makes them over- or under-wait. From a7bb524fef61f77ede01f660ffbd591e1d5837ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz?= Date: Mon, 17 Aug 2026 21:39:16 +0200 Subject: [PATCH 4/4] gh-155694: Scope HTTPPasswordMgr credentials by URL scheme (#155696) Credentials stored for an https:// URI were also matched against the corresponding http:// URI, since `reduce_uri()` discards the scheme. `HTTPPasswordMgr` and `HTTPPasswordMgrWithPriorAuth` now compare the scheme too; URIs registered without a scheme still match any scheme. --- Doc/library/urllib.request.rst | 10 +++- Lib/test/test_urllib2.py | 56 +++++++++++++++++++ Lib/urllib/request.py | 25 +++++++-- ...-07-31-16-20-17.gh-issue-155694.SsxlKG.rst | 4 ++ 4 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 95e4d2627c8b23b..9274a0c88ac4c86 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -987,8 +987,14 @@ These methods are available on :class:`HTTPPasswordMgr` and *uri* can be either a single URI, or a sequence of URIs. *realm*, *user* and *passwd* must be strings. This causes ``(user, passwd)`` to be used as - authentication tokens when authentication for *realm* and a super-URI of any of - the given URIs is given. + authentication tokens when authentication for *realm* and a super-URI of any + of the given URIs is given. If a URI includes a scheme, its credentials only + match authentication URIs with the same scheme or no scheme. A URI without a + scheme matches authentication URIs with any scheme. + + .. versionchanged:: next + Authentication credentials for URIs with a scheme are now scoped by + that scheme. .. method:: HTTPPasswordMgr.find_user_password(realm, authuri) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d2fd111f6d9de02..7efbc81a16096a6 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -270,6 +270,50 @@ def test_password_manager_default_port(self): self.assertEqual(find_user_pass("i", "http://j.example.com:80"), (None, None)) + def test_password_manager_scheme(self): + mgr = urllib.request.HTTPPasswordMgr() + mgr.add_password( + "realm", "https://example.com/", "user", "password") + + self.assertEqual( + mgr.find_user_password("realm", "https://example.com/"), + ("user", "password")) + self.assertEqual( + mgr.find_user_password("realm", "http://example.com/"), + (None, None)) + # Support an authority without a scheme. + self.assertEqual( + mgr.find_user_password("realm", "example.com"), + ("user", "password")) + # An authority without a scheme continues to match any scheme. + mgr.add_password( + "realm", "schemeless.example.com", "user", "password") + for scheme in "http", "https": + with self.subTest(scheme=scheme): + self.assertEqual( + mgr.find_user_password( + "realm", f"{scheme}://schemeless.example.com/"), + ("user", "password")) + + # A network-path reference also has no scheme. + mgr.add_password( + "realm", "//network-path.example.com/", "user", "password") + self.assertEqual( + mgr.find_user_password( + "realm", "https://network-path.example.com/"), + ("user", "password")) + + def test_password_manager_reduced_uri(self): + mgr = urllib.request.HTTPPasswordMgr() + + self.assertEqual( + mgr.reduce_uri("http://example.com/path"), + ("example.com:80", "/path")) + self.assertTrue( + mgr.is_suburi( + ("example.com", "/path"), + ("example.com", "/path/subpath"))) + class MockOpener: addheaders = [] @@ -1825,6 +1869,18 @@ def test_basic_prior_auth_auto_send(self): # expect request to be sent with auth header self.assertTrue(http_handler.has_auth_header) + def test_basic_prior_auth_different_scheme(self): + pwd_manager = HTTPPasswordMgrWithPriorAuth() + auth_handler = HTTPBasicAuthHandler(pwd_manager) + auth_handler.add_password( + None, "https://example.com/", "user", "password", + is_authenticated=True) + + request = Request("http://example.com/") + auth_handler.http_request(request) + + self.assertFalse(request.has_header("Authorization")) + def test_basic_prior_auth_send_after_first_success(self): # Auto send auth header after authentication is successful once diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 660301fef612588..9fa92659a255ed4 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -815,16 +815,17 @@ def add_password(self, realm, uri, user, passwd): self.passwd[realm] = {} for default_port in True, False: reduced_uri = tuple( - self.reduce_uri(u, default_port) for u in uri) + self._reduce_uri_with_scheme(u, default_port) for u in uri) self.passwd[realm][reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): domains = self.passwd.get(realm, {}) for default_port in True, False: - reduced_authuri = self.reduce_uri(authuri, default_port) + reduced_authuri = self._reduce_uri_with_scheme( + authuri, default_port) for uris, authinfo in domains.items(): for uri in uris: - if self.is_suburi(uri, reduced_authuri): + if self._is_suburi_with_scheme(uri, reduced_authuri): return authinfo return None, None @@ -851,6 +852,17 @@ def reduce_uri(self, uri, default_port=True): authority = "%s:%d" % (host, dport) return authority, path + def _reduce_uri_with_scheme(self, uri, default_port=True): + parts = urlsplit(uri) + scheme = parts[0] if parts[1] else None + return (scheme or None, *self.reduce_uri(uri, default_port)) + + def _is_suburi_with_scheme(self, base, test): + if (base[0] is not None and test[0] is not None and + base[0] != test[0]): + return False + return self.is_suburi(base[1:], test[1:]) + def is_suburi(self, base, test): """Check if test is below base in a URI tree @@ -896,14 +908,15 @@ def update_authenticated(self, uri, is_authenticated=False): for default_port in True, False: for u in uri: - reduced_uri = self.reduce_uri(u, default_port) + reduced_uri = self._reduce_uri_with_scheme(u, default_port) self.authenticated[reduced_uri] = is_authenticated def is_authenticated(self, authuri): for default_port in True, False: - reduced_authuri = self.reduce_uri(authuri, default_port) + reduced_authuri = self._reduce_uri_with_scheme( + authuri, default_port) for uri in self.authenticated: - if self.is_suburi(uri, reduced_authuri): + if self._is_suburi_with_scheme(uri, reduced_authuri): return self.authenticated[uri] diff --git a/Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst b/Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst new file mode 100644 index 000000000000000..dbc2119640702c9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-07-31-16-20-17.gh-issue-155694.SsxlKG.rst @@ -0,0 +1,4 @@ +Fix :cve:`2026-15806` by scoping :class:`~urllib.request.HTTPPasswordMgr` +credentials to the URL scheme, preventing credentials stored for an HTTPS +URL from being used for a matching HTTP URL, while URIs without a scheme +continue to match any scheme.