[pull] main from python:main

Doc/library/urllib.request.rst

@@ -987,8 +987,14 @@ These methods are available on :class:`HTTPPasswordMgr` and
 
    *uri* can be either a single URI, or a sequence of URIs. *realm*, *user* and
    *passwd* must be strings. This causes ``(user, passwd)`` to be used as
-   authentication tokens when authentication for *realm* and a super-URI of any of
-   the given URIs is given.
+   authentication tokens when authentication for *realm* and a super-URI of any
+   of the given URIs is given. If a URI includes a scheme, its credentials only
+   match authentication URIs with the same scheme or no scheme. A URI without a
+   scheme matches authentication URIs with any scheme.
+
+   .. versionchanged:: next
+      Authentication credentials for URIs with a scheme are now scoped by
+      that scheme.
 
 
 .. method:: HTTPPasswordMgr.find_user_password(realm, authuri)

Lib/concurrent/interpreters/_queues.py

@@ -220,12 +220,12 @@ def put(self, obj, block=True, timeout=None, *,
             timeout = int(timeout)
             if timeout < 0:
                 raise ValueError(f'timeout value must be non-negative')
-            end = time.time() + timeout
+            end = time.monotonic() + timeout
         while True:
             try:
                 _queues.put(self._id, obj, unboundop)
             except QueueFull:
-                if timeout is not None and time.time() >= end:
+                if timeout is not None and time.monotonic() >= end:
                     raise  # re-raise
                 time.sleep(_delay)
             else:
@@ -255,12 +255,12 @@ def get(self, block=True, timeout=None, *,
             timeout = int(timeout)
             if timeout < 0:
                 raise ValueError(f'timeout value must be non-negative')
-            end = time.time() + timeout
+            end = time.monotonic() + timeout
         while True:
             try:
                 obj, unboundop = _queues.get(self._id)
             except QueueEmpty:
-                if timeout is not None and time.time() >= end:
+                if timeout is not None and time.monotonic() >= end:
                     raise  # re-raise
                 time.sleep(_delay)
             else:

Lib/test/test_interpreters/test_queues.py

@@ -2,7 +2,9 @@
 import pickle
 import threading
 from textwrap import dedent
+import time
 import unittest
+from unittest import mock
 
 from test.support import import_helper, Py_DEBUG
 # Raise SkipTest if subinterpreters not supported.
@@ -354,6 +356,19 @@ def test_get_timeout(self):
         with self.assertRaises(queues.QueueEmpty):
             queue.get(HUGE_TIMEOUT, 0.1)
 
+    def test_timeout_uses_monotonic_clock(self):
+        # gh-153005: the deadline must be computed from the monotonic clock,
+        # since the wall clock can be adjusted while the call is blocked.
+        queue = queues.create(1)
+        with mock.patch.object(queues, 'time', wraps=time) as fake_time:
+            with self.assertRaises(queues.QueueEmpty):
+                queue.get(timeout=0)
+            queue.put(None)
+            with self.assertRaises(queues.QueueFull):
+                queue.put(None, timeout=0)
+        fake_time.monotonic.assert_called()
+        fake_time.time.assert_not_called()
+
     def test_get_nowait(self):
         queue = queues.create()
         with self.assertRaises(queues.QueueEmpty):

Lib/test/test_io/test_textio.py

@@ -1610,6 +1610,49 @@ def make_text(buffer):
             wrapper.write('x')
             self.assertRaisesRegex(ValueError, "detached", wrapper.read)
 
+    def test_reentrant_seek_during_tell(self):
+        # gh-153539: reading short of _CHUNK_SIZE leaves residual bytes in the
+        # snapshot, so tell() re-decodes and calls the decoder's getstate(); a
+        # reentrant seek() there must not free the snapshot tell() still uses.
+        # C-only: _pyio binds next_input as a strong local and cannot crash.
+        wrapper = None
+        armed = False
+
+        class ReentrantDecoder(codecs.IncrementalDecoder):
+            def decode(self, input, final=False):
+                return bytes(input).decode("latin-1")
+            def getstate(self):
+                nonlocal armed
+                if wrapper is not None and armed:
+                    armed = False
+                    wrapper.seek(0)
+                return (b"", 0)
+            def setstate(self, state):
+                pass
+
+        def search(name):
+            if name != "reentrant_tell_test":
+                return None
+            return codecs.CodecInfo(
+                name=name,
+                encode=lambda s, e='strict': (s.encode("latin-1"), len(s)),
+                decode=lambda b, e='strict': (bytes(b).decode("latin-1"), len(b)),
+                incrementaldecoder=ReentrantDecoder)
+
+        codecs.register(search)
+        self.addCleanup(codecs.unregister, search)
+        raw = self.BytesIO(b"abcdefghijklmnop" * 8)
+        wrapper = self.TextIOWrapper(self.BufferedReader(raw),
+                                     encoding="reentrant_tell_test", newline="")
+        wrapper._CHUNK_SIZE = 8
+        wrapper.read(5)
+        armed = True
+        self.assertIsInstance(wrapper.tell(), int)
+        # tell() at the snapshot boundary takes the early return that owns and
+        # must release next_input; exercise it too (leak-checked under -R).
+        wrapper.seek(0)
+        self.assertIsInstance(wrapper.tell(), int)
+
 
 class PyTextIOWrapperTest(TextIOWrapperTest, PyTestCase):
     shutdown_error = "LookupError: unknown encoding: ascii"

Lib/test/test_urllib2.py

@@ -270,6 +270,50 @@ def test_password_manager_default_port(self):
         self.assertEqual(find_user_pass("i", "http://j.example.com:80"),
                          (None, None))
 
+    def test_password_manager_scheme(self):
+        mgr = urllib.request.HTTPPasswordMgr()
+        mgr.add_password(
+            "realm", "https://example.com/", "user", "password")
+
+        self.assertEqual(
+            mgr.find_user_password("realm", "https://example.com/"),
+            ("user", "password"))
+        self.assertEqual(
+            mgr.find_user_password("realm", "http://example.com/"),
+            (None, None))
+        # Support an authority without a scheme.
+        self.assertEqual(
+            mgr.find_user_password("realm", "example.com"),
+            ("user", "password"))
+        # An authority without a scheme continues to match any scheme.
+        mgr.add_password(
+            "realm", "schemeless.example.com", "user", "password")
+        for scheme in "http", "https":
+            with self.subTest(scheme=scheme):
+                self.assertEqual(
+                    mgr.find_user_password(
+                        "realm", f"{scheme}://schemeless.example.com/"),
+                    ("user", "password"))
+
+        # A network-path reference also has no scheme.
+        mgr.add_password(
+            "realm", "//network-path.example.com/", "user", "password")
+        self.assertEqual(
+            mgr.find_user_password(
+                "realm", "https://network-path.example.com/"),
+            ("user", "password"))
+
+    def test_password_manager_reduced_uri(self):
+        mgr = urllib.request.HTTPPasswordMgr()
+
+        self.assertEqual(
+            mgr.reduce_uri("http://example.com/path"),
+            ("example.com:80", "/path"))
+        self.assertTrue(
+            mgr.is_suburi(
+                ("example.com", "/path"),
+                ("example.com", "/path/subpath")))
+
 
 class MockOpener:
     addheaders = []
@@ -1825,6 +1869,18 @@ def test_basic_prior_auth_auto_send(self):
         # expect request to be sent with auth header
         self.assertTrue(http_handler.has_auth_header)
 
+    def test_basic_prior_auth_different_scheme(self):
+        pwd_manager = HTTPPasswordMgrWithPriorAuth()
+        auth_handler = HTTPBasicAuthHandler(pwd_manager)
+        auth_handler.add_password(
+            None, "https://example.com/", "user", "password",
+            is_authenticated=True)
+
+        request = Request("http://example.com/")
+        auth_handler.http_request(request)
+
+        self.assertFalse(request.has_header("Authorization"))
+
     def test_basic_prior_auth_send_after_first_success(self):
         # Auto send auth header after authentication is successful once
 

Lib/test/test_warnings/__init__.py

@@ -13,6 +13,7 @@
 import unittest
 from test import support
 from test.support import import_helper
+from test.support import isolation
 from test.support import os_helper
 from test.support import warnings_helper
 from test.support import force_not_colorized
@@ -1518,49 +1519,74 @@ class PyCatchWarningTests(CatchWarningTests, unittest.TestCase):
     module = py_warnings
 
 
-class EnvironmentVariableTests(BaseTest):
+_NONASCII_WARNOPTION = 'ignore:DeprecationWarning' + os_helper.FS_NONASCII
+
+
+class WarnOptionsTests(unittest.TestCase):
+    """Tests of the -W option and the PYTHONWARNINGS environment variable.
+
+    sys.warnoptions is set by the interpreter, so these tests do not depend
+    on the used implementation of the warnings module.
+    """
 
+    @isolation.runInSubprocess(
+        env={'PYTHONWARNINGS': 'ignore::DeprecationWarning',
+             'PYTHONDEVMODE': ''})
     def test_single_warning(self):
-        rc, stdout, stderr = assert_python_ok("-c",
-            "import sys; sys.stdout.write(str(sys.warnoptions))",
-            PYTHONWARNINGS="ignore::DeprecationWarning",
-            PYTHONDEVMODE="")
-        self.assertEqual(stdout, b"['ignore::DeprecationWarning']")
+        self.assertEqual(sys.warnoptions, ['ignore::DeprecationWarning'])
 
+    @isolation.runInSubprocess(
+        env={'PYTHONWARNINGS': 'ignore::DeprecationWarning,'
+                               'ignore::UnicodeWarning',
+             'PYTHONDEVMODE': ''})
     def test_comma_separated_warnings(self):
-        rc, stdout, stderr = assert_python_ok("-c",
-            "import sys; sys.stdout.write(str(sys.warnoptions))",
-            PYTHONWARNINGS="ignore::DeprecationWarning,ignore::UnicodeWarning",
-            PYTHONDEVMODE="")
-        self.assertEqual(stdout,
-            b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']")
+        self.assertEqual(sys.warnoptions, ['ignore::DeprecationWarning',
+                                           'ignore::UnicodeWarning'])
 
-    @force_not_colorized
+    @isolation.runInSubprocess(
+        options=['-Wignore::UnicodeWarning'],
+        env={'PYTHONWARNINGS': 'ignore::DeprecationWarning',
+             'PYTHONDEVMODE': ''})
     def test_envvar_and_command_line(self):
-        rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c",
-            "import sys; sys.stdout.write(str(sys.warnoptions))",
-            PYTHONWARNINGS="ignore::DeprecationWarning",
-            PYTHONDEVMODE="")
-        self.assertEqual(stdout,
-            b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']")
+        self.assertEqual(sys.warnoptions, ['ignore::DeprecationWarning',
+                                           'ignore::UnicodeWarning'])
+
+    @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii',
+                         'requires non-ascii filesystemencoding')
+    @isolation.runInSubprocess(
+        env={'PYTHONWARNINGS': _NONASCII_WARNOPTION,
+             'PYTHONDEVMODE': ''})
+    def test_nonascii(self):
+        self.assertEqual(sys.warnoptions, [_NONASCII_WARNOPTION])
+
+
+class EnvironmentVariableTests(BaseTest):
+
+    def prepare_code(self, code):
+        """Make the subprocess use the tested implementation."""
+        if self.module is py_warnings:
+            # Disable the warnings acceleration module in the subprocess.
+            code = ("import sys; sys.modules.pop('warnings', None); "
+                    "sys.modules['_warnings'] = None; ") + code
+        return code
 
     @force_not_colorized
     def test_conflicting_envvar_and_command_line(self):
-        rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c",
+        code = self.prepare_code(
             "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); "
-            "warnings.warn('Message', DeprecationWarning)",
+            "warnings.warn('Message', DeprecationWarning)")
+        rc, stdout, stderr = assert_python_failure(
+            "-Werror::DeprecationWarning", "-c", code,
             PYTHONWARNINGS="default::DeprecationWarning",
             PYTHONDEVMODE="")
         self.assertEqual(stdout,
             b"['default::DeprecationWarning', 'error::DeprecationWarning']")
-        self.assertEqual(stderr.splitlines(),
-            [b"Traceback (most recent call last):",
-             b"  File \"<string>\", line 1, in <module>",
-             b'    import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w'
-             b"arn('Message', DeprecationWarning)",
-             b'                                                                  ~~~~~~~~~~'
-             b'~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^',
-             b"DeprecationWarning: Message"])
+        # The traceback of the Python implementation contains additional
+        # frames, so only the ends of the traceback are checked.
+        lines = stderr.splitlines()
+        self.assertEqual(lines[0], b"Traceback (most recent call last):")
+        self.assertEqual(lines[1], b'  File "<string>", line 1, in <module>')
+        self.assertEqual(lines[-1], b"DeprecationWarning: Message")
 
     def test_default_filter_configuration(self):
         pure_python_api = self.module is py_warnings
@@ -1580,30 +1606,15 @@ def test_default_filter_configuration(self):
             ]
         expected_output = [str(f).encode() for f in expected_default_filters]
 
-        if pure_python_api:
-            # Disable the warnings acceleration module in the subprocess
-            code = "import sys; sys.modules.pop('warnings', None); sys.modules['_warnings'] = None; "
-        else:
-            code = ""
-        code += "import warnings; [print(f) for f in warnings._get_filters()]"
+        code = self.prepare_code(
+            "import warnings; [print(f) for f in warnings._get_filters()]")
 
         rc, stdout, stderr = assert_python_ok("-c", code, __isolated=True)
         stdout_lines = [line.strip() for line in stdout.splitlines()]
         self.maxDiff = None
         self.assertEqual(stdout_lines, expected_output)
 
 
-    @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii',
-                         'requires non-ascii filesystemencoding')
-    def test_nonascii(self):
-        PYTHONWARNINGS="ignore:DeprecationWarning" + os_helper.FS_NONASCII
-        rc, stdout, stderr = assert_python_ok("-c",
-            "import sys; sys.stdout.write(str(sys.warnoptions))",
-            PYTHONIOENCODING="utf-8",
-            PYTHONWARNINGS=PYTHONWARNINGS,
-            PYTHONDEVMODE="")
-        self.assertEqual(stdout, str([PYTHONWARNINGS]).encode())
-
 class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase):
     module = c_warnings
 

Lib/urllib/request.py

@@ -815,16 +815,17 @@ def add_password(self, realm, uri, user, passwd):
             self.passwd[realm] = {}
         for default_port in True, False:
             reduced_uri = tuple(
-                self.reduce_uri(u, default_port) for u in uri)
+                self._reduce_uri_with_scheme(u, default_port) for u in uri)
             self.passwd[realm][reduced_uri] = (user, passwd)
 
     def find_user_password(self, realm, authuri):
         domains = self.passwd.get(realm, {})
         for default_port in True, False:
-            reduced_authuri = self.reduce_uri(authuri, default_port)
+            reduced_authuri = self._reduce_uri_with_scheme(
+                authuri, default_port)
             for uris, authinfo in domains.items():
                 for uri in uris:
-                    if self.is_suburi(uri, reduced_authuri):
+                    if self._is_suburi_with_scheme(uri, reduced_authuri):
                         return authinfo
         return None, None
 
@@ -851,6 +852,17 @@ def reduce_uri(self, uri, default_port=True):
                 authority = "%s:%d" % (host, dport)
         return authority, path
 
+    def _reduce_uri_with_scheme(self, uri, default_port=True):
+        parts = urlsplit(uri)
+        scheme = parts[0] if parts[1] else None
+        return (scheme or None, *self.reduce_uri(uri, default_port))
+
+    def _is_suburi_with_scheme(self, base, test):
+        if (base[0] is not None and test[0] is not None and
+                base[0] != test[0]):
+            return False
+        return self.is_suburi(base[1:], test[1:])
+
     def is_suburi(self, base, test):
         """Check if test is below base in a URI tree
 
@@ -896,14 +908,15 @@ def update_authenticated(self, uri, is_authenticated=False):
 
         for default_port in True, False:
             for u in uri:
-                reduced_uri = self.reduce_uri(u, default_port)
+                reduced_uri = self._reduce_uri_with_scheme(u, default_port)
                 self.authenticated[reduced_uri] = is_authenticated
 
     def is_authenticated(self, authuri):
         for default_port in True, False:
-            reduced_authuri = self.reduce_uri(authuri, default_port)
+            reduced_authuri = self._reduce_uri_with_scheme(
+                authuri, default_port)
             for uri in self.authenticated:
-                if self.is_suburi(uri, reduced_authuri):
+                if self._is_suburi_with_scheme(uri, reduced_authuri):
                     return self.authenticated[uri]