diff --git a/Lib/locale.py b/Lib/locale.py index 25efff5b8568543..03fc1dd844dd8b9 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -921,6 +921,11 @@ def getpreferredencoding(do_setlocale=True): # # removed 'el_gr@euro' # removed 'uz_uz@cyrillic' +# +# gh-151316: +# Prefer glibc UTF-8 for bare locales that are UTF-8-only in SUPPORTED. +# X11 locale.alias still maps these to obsolete codesets and would otherwise +# override glibc during makelocalealias regeneration. locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -965,7 +970,7 @@ def getpreferredencoding(do_setlocale=True): 'ast_es': 'ast_ES.ISO8859-15', 'ayc_pe': 'ayc_PE.UTF-8', 'az': 'az_AZ.ISO8859-9E', - 'az_az': 'az_AZ.ISO8859-9E', + 'az_az': 'az_AZ.UTF-8', 'az_az.iso88599e': 'az_AZ.ISO8859-9E', 'az_ir': 'az_IR.UTF-8', 'be': 'be_BY.CP1251', @@ -1039,7 +1044,7 @@ def getpreferredencoding(do_setlocale=True): 'de_ch': 'de_CH.ISO8859-1', 'de_de': 'de_DE.ISO8859-1', 'de_it': 'de_IT.UTF-8', - 'de_li': 'de_LI.ISO8859-1', + 'de_li': 'de_LI.UTF-8', 'de_lu': 'de_LU.ISO8859-1', 'deutsch': 'de_DE.ISO8859-1', 'doi_in': 'doi_IN.UTF-8', @@ -1065,8 +1070,8 @@ def getpreferredencoding(do_setlocale=True): 'en_gb': 'en_GB.ISO8859-1', 'en_hk': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', - 'en_il': 'en_IL.ISO8859-1', - 'en_in': 'en_IN.ISO8859-1', + 'en_il': 'en_IL.UTF-8', + 'en_in': 'en_IN.UTF-8', 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', 'en_ph': 'en_PH.ISO8859-1', @@ -1085,7 +1090,7 @@ def getpreferredencoding(do_setlocale=True): 'english_united-states': 'en_US.ISO8859-1', 'english_united-states.437': 'C', 'english_us': 'en_US.ISO8859-1', - 'eo': 'eo_XX.ISO8859-3', + 'eo': 'eo.UTF-8', 'eo.utf8': 'eo.UTF-8', 'eo_eo': 'eo_EO.ISO8859-3', 'eo_us.utf8': 'eo_US.UTF-8', @@ -1096,7 +1101,7 @@ def getpreferredencoding(do_setlocale=True): 'es_cl': 'es_CL.ISO8859-1', 'es_co': 'es_CO.ISO8859-1', 'es_cr': 'es_CR.ISO8859-1', - 'es_cu': 'es_CU.ISO8859-1', + 'es_cu': 'es_CU.UTF-8', 'es_do': 'es_DO.ISO8859-1', 'es_ec': 'es_EC.ISO8859-1', 'es_es': 'es_ES.ISO8859-1', @@ -1167,7 +1172,7 @@ def getpreferredencoding(do_setlocale=True): 'he_il': 'he_IL.ISO8859-8', 'hebrew': 'he_IL.ISO8859-8', 'hi': 'hi_IN.ISCII-DEV', - 'hi_in': 'hi_IN.ISCII-DEV', + 'hi_in': 'hi_IN.UTF-8', 'hi_in.isciidev': 'hi_IN.ISCII-DEV', 'hif_fj': 'hif_FJ.UTF-8', 'hne': 'hne_IN.UTF-8', @@ -1205,7 +1210,7 @@ def getpreferredencoding(do_setlocale=True): 'it_it': 'it_IT.ISO8859-1', 'italian': 'it_IT.ISO8859-1', 'iu': 'iu_CA.NUNACOM-8', - 'iu_ca': 'iu_CA.NUNACOM-8', + 'iu_ca': 'iu_CA.UTF-8', 'iu_ca.nunacom8': 'iu_CA.NUNACOM-8', 'iw': 'he_IL.ISO8859-8', 'iw_il': 'he_IL.ISO8859-8', @@ -1255,7 +1260,7 @@ def getpreferredencoding(do_setlocale=True): 'lithuanian': 'lt_LT.ISO8859-13', 'ln_cd': 'ln_CD.UTF-8', 'lo': 'lo_LA.MULELAO-1', - 'lo_la': 'lo_LA.MULELAO-1', + 'lo_la': 'lo_LA.UTF-8', 'lo_la.cp1133': 'lo_LA.IBM-CP1133', 'lo_la.ibmcp1133': 'lo_LA.IBM-CP1133', 'lo_la.mulelao1': 'lo_LA.MULELAO-1', @@ -1313,9 +1318,9 @@ def getpreferredencoding(do_setlocale=True): 'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1', 'norwegian': 'no_NO.ISO8859-1', 'nr': 'nr_ZA.ISO8859-1', - 'nr_za': 'nr_ZA.ISO8859-1', + 'nr_za': 'nr_ZA.UTF-8', 'nso': 'nso_ZA.ISO8859-15', - 'nso_za': 'nso_ZA.ISO8859-15', + 'nso_za': 'nso_ZA.UTF-8', 'ny': 'ny_NO.ISO8859-1', 'ny_no': 'ny_NO.ISO8859-1', 'nynorsk': 'nn_NO.ISO8859-1', @@ -1362,7 +1367,7 @@ def getpreferredencoding(do_setlocale=True): 'rumanian': 'ro_RO.ISO8859-2', 'russian': 'ru_RU.ISO8859-5', 'rw': 'rw_RW.ISO8859-1', - 'rw_rw': 'rw_RW.ISO8859-1', + 'rw_rw': 'rw_RW.UTF-8', 'sa_in': 'sa_IN.UTF-8', 'sah_ru': 'sah_RU.UTF-8', 'sat_in': 'sat_IN.UTF-8', @@ -1427,7 +1432,7 @@ def getpreferredencoding(do_setlocale=True): 'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8', 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', - 'ss_za': 'ss_ZA.ISO8859-1', + 'ss_za': 'ss_ZA.UTF-8', 'ssy_er': 'ssy_ER.UTF-8', 'st': 'st_ZA.ISO8859-1', 'st_za': 'st_ZA.ISO8859-1', @@ -1441,7 +1446,7 @@ def getpreferredencoding(do_setlocale=True): 'syr': 'syr.UTF-8', 'szl_pl': 'szl_PL.UTF-8', 'ta': 'ta_IN.TSCII-0', - 'ta_in': 'ta_IN.TSCII-0', + 'ta_in': 'ta_IN.UTF-8', 'ta_in.tscii': 'ta_IN.TSCII-0', 'ta_in.tscii0': 'ta_IN.TSCII-0', 'ta_lk': 'ta_LK.UTF-8', @@ -1463,7 +1468,7 @@ def getpreferredencoding(do_setlocale=True): 'tl': 'tl_PH.ISO8859-1', 'tl_ph': 'tl_PH.ISO8859-1', 'tn': 'tn_ZA.ISO8859-15', - 'tn_za': 'tn_ZA.ISO8859-15', + 'tn_za': 'tn_ZA.UTF-8', 'to_to': 'to_TO.UTF-8', 'tok': 'tok.UTF-8', 'tpi_pg': 'tpi_PG.UTF-8', @@ -1471,9 +1476,9 @@ def getpreferredencoding(do_setlocale=True): 'tr_cy': 'tr_CY.ISO8859-9', 'tr_tr': 'tr_TR.ISO8859-9', 'ts': 'ts_ZA.ISO8859-1', - 'ts_za': 'ts_ZA.ISO8859-1', + 'ts_za': 'ts_ZA.UTF-8', 'tt': 'tt_RU.TATAR-CYR', - 'tt_ru': 'tt_RU.TATAR-CYR', + 'tt_ru': 'tt_RU.UTF-8', 'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR', 'tt_ru@iqtelif': 'tt_RU.UTF-8@iqtelif', 'turkish': 'tr_TR.ISO8859-9', @@ -1485,13 +1490,13 @@ def getpreferredencoding(do_setlocale=True): 'unm_us': 'unm_US.UTF-8', 'ur': 'ur_PK.CP1256', 'ur_in': 'ur_IN.UTF-8', - 'ur_pk': 'ur_PK.CP1256', + 'ur_pk': 'ur_PK.UTF-8', 'uz': 'uz_UZ.UTF-8', 'uz_uz': 'uz_UZ.UTF-8', 've': 've_ZA.UTF-8', 've_za': 've_ZA.UTF-8', 'vi': 'vi_VN.TCVN', - 'vi_vn': 'vi_VN.TCVN', + 'vi_vn': 'vi_VN.UTF-8', 'vi_vn.tcvn': 'vi_VN.TCVN', 'vi_vn.tcvn5712': 'vi_VN.TCVN', 'vi_vn.viscii': 'vi_VN.VISCII', diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 8057c35f540841b..9a929bc6d77a441 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -642,6 +642,53 @@ def test_getlocale_with_modifier(self, localename, localetuple): self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple) +# Bare aliases that must stay on UTF-8 after X11 locale.alias regeneration +# (gh-151316). Keep in sync with apply_x11_locales_patch() in +# Tools/i18n/makelocalealias.py. +_UTF8_X11_OVERRIDES = ( + ('az_az', 'az_AZ.UTF-8'), + ('de_li', 'de_LI.UTF-8'), + ('en_il', 'en_IL.UTF-8'), + ('en_in', 'en_IN.UTF-8'), + ('eo', 'eo.UTF-8'), + ('es_cu', 'es_CU.UTF-8'), + ('hi_in', 'hi_IN.UTF-8'), + ('iu_ca', 'iu_CA.UTF-8'), + ('lo_la', 'lo_LA.UTF-8'), + ('nr_za', 'nr_ZA.UTF-8'), + ('nso_za', 'nso_ZA.UTF-8'), + ('rw_rw', 'rw_RW.UTF-8'), + ('ss_za', 'ss_ZA.UTF-8'), + ('ta_in', 'ta_IN.UTF-8'), + ('tn_za', 'tn_ZA.UTF-8'), + ('ts_za', 'ts_ZA.UTF-8'), + ('tt_ru', 'tt_RU.UTF-8'), + ('ur_pk', 'ur_PK.UTF-8'), + ('vi_vn', 'vi_VN.UTF-8'), +) + + +class TestHardcodedLocaleReplacements(unittest.TestCase): + """UTF-8 preferences that override X11 legacy codesets. + + See https://github.com/python/cpython/issues/151316 + """ + + @support.subTests('key,expected', _UTF8_X11_OVERRIDES) + def test_locale_alias_entries(self, key, expected): + self.assertEqual(locale.locale_alias[key], expected) + + @support.subTests('key,expected', _UTF8_X11_OVERRIDES) + def test_normalize(self, key, expected): + self.assertEqual(locale.normalize(key), expected) + self.assertEqual(locale.normalize(key.upper()), expected) + + @support.subTests('key,expected', _UTF8_X11_OVERRIDES) + def test_parse_localename(self, key, expected): + lang, encoding = expected.split('.') + self.assertEqual(locale._parse_localename(key), (lang, encoding)) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is diff --git a/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst b/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst new file mode 100644 index 000000000000000..5223d7ff4d19954 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst @@ -0,0 +1,5 @@ +:mod:`locale` now prefers UTF-8 for bare locale aliases that are +UTF-8-only in glibc (for example ``en_IN``, ``hi_IN``, ``es_CU``) +instead of obsolete X11 codesets. This restores +``setlocale(getlocale())`` round-trips on modern systems where those +locales have no legacy encoding. diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index f825862ffdf350a..99442a3d47f4dd6 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -98,6 +98,25 @@ def parse_glibc_supported(filename): data[locale] = alias return data +def apply_x11_locales_patch(data, glibc_data): + """Restore glibc UTF-8 defaults that X11 locale.alias overwrote. + + Regeneration updates glibc SUPPORTED first, then X11 locale.alias. + X11 still maps several locales that are UTF-8-only in modern glibc to + obsolete encodings (for example en_IN -> en_IN.ISO8859-1). Prefer the + glibc UTF-8 mapping for those bare keys (gh-151316). + + de_LI is special: glibc only ships de_LI.UTF-8 (no bare SUPPORTED line), + so the X11 de_LI.ISO8859-1 mapping would otherwise stick. + """ + for key, value in glibc_data.items(): + if data.get(key) == value: + continue + if value.split('@')[0].endswith('.UTF-8'): + data[key] = value + data['de_li'] = 'de_LI.UTF-8' + return data + def pprint(data): items = sorted(data.items()) for k, v in items: @@ -149,11 +168,13 @@ def check(data): args = parser.parse_args() data = locale.locale_alias.copy() - data.update(parse_glibc_supported(args.glibc_supported)) + glibc_data = parse_glibc_supported(args.glibc_supported) + data.update(glibc_data) data.update(parse(args.locale_alias)) # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist # on all platforms. data['c.utf8'] = 'C.UTF-8' + data = apply_x11_locales_patch(data, glibc_data) while True: # Repeat optimization while the size is decreased. n = len(data)