gh-155247: Use raw string content for the extensions cache key

The key of the extensions cache was built with _PyUnicode_AsUTF8NoNUL(),
so importing an extension module failed with UnicodeEncodeError if its
path contained characters unencodable in UTF-8, e.g. surrogate escapes of
an undecodable file name.  The raw content of the strings is now used.

The rejection of embedded null characters, which was a side effect of the
UTF-8 encoding, is now explicit in the extension module loader.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: serhiy-storchaka

Lib/test/test_import/__init__.py

@@ -1262,6 +1262,34 @@ class Spec2:
                 origin = "a\x00b"
             _imp.create_dynamic(Spec2())
 
+    @unittest.skipUnless(_testsinglephase is not None,
+                         'requires _testsinglephase')
+    @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE,
+                         'requires undecodable file names')
+    def test_import_from_undecodable_path(self):
+        # gh-155247: the path of the extension module is not encodable
+        # in UTF-8.
+        origin = _testsinglephase.__file__
+        # The module is cached by its path, so restore it afterwards.
+        self.addCleanup(restore__testsinglephase)
+        with os_helper.temp_dir() as tempdir:
+            subdir = os.path.join(os.fsencode(tempdir),
+                                  os_helper.TESTFN_UNDECODABLE)
+            try:
+                os.mkdir(subdir)
+            except OSError:
+                self.skipTest('undecodable paths are not supported')
+            path = os.path.join(subdir, os.fsencode(os.path.basename(origin)))
+            shutil.copyfile(origin, path)
+            path = os.fsdecode(path)
+            spec = importlib.util.spec_from_file_location('_testsinglephase',
+                                                          path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            self.assertEqual(module.__name__, '_testsinglephase')
+            self.assertEqual(module.__file__, path)
+            _testinternalcapi.clear_extension('_testsinglephase', path)
+
     def test_create_builtin(self):
         class Spec:
             pass

Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst

@@ -0,0 +1,4 @@
+Fix importing an extension module whose path contains characters unencodable
+in UTF-8, e.g. undecodable bytes of a file name.  Previously it failed with
+:exc:`UnicodeEncodeError`, which made it impossible to build or run Python in
+a directory with such name.

Python/import.c

@@ -1276,50 +1276,66 @@ del_extensions_cache_value(void *raw)
     }
 }
 
+/* The key of the extensions cache: the raw content of two strings.
+
+   The UTF-8 encoding is not used, because the strings can contain lone
+   surrogates, e.g. a file name undecodable in the filesystem encoding. */
+struct hashtable_key {
+    size_t size;        /* the total size of the key */
+    unsigned char kind1;
+    unsigned char kind2;
+    Py_ssize_t len1;
+    /* followed by the raw content of both strings */
+};
+
 static void *
-hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep)
+hashtable_key_from_2_strings(PyObject *str1, PyObject *str2)
 {
-    const char *str1_data = _PyUnicode_AsUTF8NoNUL(str1);
-    const char *str2_data = _PyUnicode_AsUTF8NoNUL(str2);
-    if (str1_data == NULL || str2_data == NULL) {
-        return NULL;
-    }
-    Py_ssize_t str1_len = strlen(str1_data);
-    Py_ssize_t str2_len = strlen(str2_data);
+    Py_ssize_t len1 = PyUnicode_GET_LENGTH(str1);
+    Py_ssize_t len2 = PyUnicode_GET_LENGTH(str2);
+    int kind1 = PyUnicode_KIND(str1);
+    int kind2 = PyUnicode_KIND(str2);
+    size_t size1 = (size_t)len1 * kind1;
+    size_t size2 = (size_t)len2 * kind2;
 
-    /* Make sure sep and the NULL byte won't cause an overflow. */
-    assert(SIZE_MAX - str1_len - str2_len > 2);
-    size_t size = str1_len + 1 + str2_len + 1;
+    assert(SIZE_MAX - sizeof(struct hashtable_key) - size1 > size2);
+    size_t size = sizeof(struct hashtable_key) + size1 + size2;
 
     // XXX Use a buffer if it's a temp value (every case but "set").
-    char *key = PyMem_RawMalloc(size);
+    struct hashtable_key *key = PyMem_RawMalloc(size);
     if (key == NULL) {
         PyErr_NoMemory();
         return NULL;
     }
 
-    memcpy(key, str1_data, str1_len);
-    key[str1_len] = sep;
-    memcpy(key + str1_len + 1, str2_data, str2_len);
-    key[size - 1] = '\0';
-    assert(strlen(key) == size - 1);
+    key->size = size;
+    key->kind1 = (unsigned char)kind1;
+    key->kind2 = (unsigned char)kind2;
+    key->len1 = len1;
+    char *data = (char *)(key + 1);
+    memcpy(data, PyUnicode_DATA(str1), size1);
+    memcpy(data + size1, PyUnicode_DATA(str2), size2);
     return key;
 }
 
 static Py_uhash_t
-hashtable_hash_str(const void *key)
+hashtable_hash_key(const void *key)
 {
-    return Py_HashBuffer(key, strlen((const char *)key));
+    return Py_HashBuffer(key, ((const struct hashtable_key *)key)->size);
 }
 
 static int
-hashtable_compare_str(const void *key1, const void *key2)
+hashtable_compare_key(const void *key1, const void *key2)
 {
-    return strcmp((const char *)key1, (const char *)key2) == 0;
+    size_t size = ((const struct hashtable_key *)key1)->size;
+    if (size != ((const struct hashtable_key *)key2)->size) {
+        return 0;
+    }
+    return memcmp(key1, key2, size) == 0;
 }
 
 static void
-hashtable_destroy_str(void *ptr)
+hashtable_destroy_key(void *ptr)
 {
     PyMem_RawFree(ptr);
 }
@@ -1359,16 +1375,15 @@ _find_cached_def(PyModuleDef *def)
 }
 #endif
 
-#define HTSEP ':'
 
 static int
 _extensions_cache_init(void)
 {
     _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree};
     EXTENSIONS.hashtable = _Py_hashtable_new_full(
-        hashtable_hash_str,
-        hashtable_compare_str,
-        hashtable_destroy_str,  // key
+        hashtable_hash_key,
+        hashtable_compare_key,
+        hashtable_destroy_key,  // key
         del_extensions_cache_value,  // value
         &alloc
     );
@@ -1386,7 +1401,7 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name,
     if (EXTENSIONS.hashtable == NULL) {
         return NULL;
     }
-    void *key = hashtable_key_from_2_strings(path, name, HTSEP);
+    void *key = hashtable_key_from_2_strings(path, name);
     if (key == NULL) {
         return NULL;
     }
@@ -1396,7 +1411,7 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name,
         *p_key = key;
     }
     else {
-        hashtable_destroy_str(key);
+        hashtable_destroy_key(key);
     }
     return entry;
 }
@@ -1534,7 +1549,7 @@ _extensions_cache_set(PyObject *path, PyObject *name,
 finally_oldvalue:
     extensions_lock_release();
     if (key != NULL) {
-        hashtable_destroy_str(key);
+        hashtable_destroy_key(key);
     }
 
     return value;
@@ -1578,7 +1593,6 @@ _extensions_cache_clear_all(void)
     EXTENSIONS.hashtable = NULL;
 }
 
-#undef HTSEP
 
 
 static bool

Python/importdl.c

@@ -8,6 +8,7 @@
 #include "pycore_moduleobject.h"  // _PyModule_GetDefOrNull()
 #include "pycore_pyerrors.h"      // _PyErr_FormatFromCause()
 #include "pycore_runtime.h"       // _Py_ID()
+#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL()
 
 
 /***********************************/
@@ -117,7 +118,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info,
         return -1;
     }
 
-    info.newcontext = PyUnicode_AsUTF8(info.name);
+    info.newcontext = _PyUnicode_AsUTF8NoNUL(info.name);
     if (info.newcontext == NULL) {
         _Py_ext_module_loader_info_clear(&info);
         return -1;
@@ -130,6 +131,12 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info,
             _Py_ext_module_loader_info_clear(&info);
             return -1;
         }
+        if (PyUnicode_FindChar(filename, 0, 0,
+                               PyUnicode_GET_LENGTH(filename), 1) != -1) {
+            PyErr_SetString(PyExc_ValueError, "embedded null character");
+            _Py_ext_module_loader_info_clear(&info);
+            return -1;
+        }
         info.filename = Py_NewRef(filename);
 
 #ifndef MS_WINDOWS