From b959df926853f6361a8c68f54711142f902c9eff Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Tue, 11 Aug 2026 16:23:49 +0530 Subject: [PATCH 1/3] gh-151316: Prefer UTF-8 for the en_IN locale alias X11 locale.alias mapped en_IN to ISO8859-1, so getlocale() invented a codeset that does not exist on modern UTF-8-only systems and broke setlocale(getlocale()) round-trips. Prefer en_IN.UTF-8 (mirroring other modernized aliases) and cover the round-trip in tests. Signed-off-by: Sankalp Thakur --- Lib/locale.py | 7 +++- Lib/test/test_locale.py | 39 +++++++++++++++++++ ...-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst | 4 ++ Tools/i18n/makelocalealias.py | 3 ++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst diff --git a/Lib/locale.py b/Lib/locale.py index 25efff5b8568543..e37fe73ae039d93 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -921,6 +921,11 @@ def getpreferredencoding(do_setlocale=True): # # removed 'el_gr@euro' # removed 'uz_uz@cyrillic' +# +# gh-151316: +# Prefer UTF-8 for en_IN. X11 locale.alias maps en_IN to ISO8859-1 and that +# entry overrides glibc's en_IN/UTF-8 during makelocalealias regeneration, +# so hardcode the modern codeset (mirroring the c.utf8 carve-out). locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -1066,7 +1071,7 @@ def getpreferredencoding(do_setlocale=True): 'en_hk': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', 'en_il': 'en_IL.ISO8859-1', - 'en_in': 'en_IN.ISO8859-1', + 'en_in': 'en_IN.UTF-8', 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', 'en_ph': 'en_PH.ISO8859-1', diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 8057c35f540841b..3570f96fdfe7786 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -417,6 +417,13 @@ def test_english(self): self.check('english', 'en_EN.ISO8859-1') self.check('english_uk.ascii', 'en_GB.ISO8859-1') + def test_en_in_utf8(self): + # gh-151316: en_IN is UTF-8 on modern glibc; do not invent ISO8859-1. + self.check('en_IN', 'en_IN.UTF-8') + self.check('en_in', 'en_IN.UTF-8') + self.assertEqual(locale._parse_localename('en_IN'), ('en_IN', 'UTF-8')) + self.assertEqual(locale._parse_localename('en_in'), ('en_IN', 'UTF-8')) + def test_hyphenated_encoding(self): self.check('az_AZ.iso88599e', 'az_AZ.ISO8859-9E') self.check('az_AZ.ISO8859-9E', 'az_AZ.ISO8859-9E') @@ -642,6 +649,38 @@ def test_getlocale_with_modifier(self, localename, localetuple): self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple) +class TestEnINLocale(unittest.TestCase): + """gh-151316: en_IN must round-trip without inventing ISO8859-1.""" + + def setUp(self): + self.oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, self.oldlocale) + + def test_getlocale_setlocale_roundtrip(self): + try: + locale.setlocale(locale.LC_CTYPE, 'en_IN') + except locale.Error as exc: + self.skipTest(str(exc)) + loc = locale.getlocale(locale.LC_CTYPE) + self.assertEqual(loc[0], 'en_IN') + self.assertNotEqual(loc[1], 'ISO8859-1') + locale.setlocale(locale.LC_CTYPE, loc) + self.assertEqual(locale.getlocale(locale.LC_CTYPE), loc) + + def test_setlocale_from_getlocale_tuple(self): + # Reproduces the issue report: setlocale(LC_*, getlocale()). + try: + locale.setlocale(locale.LC_CTYPE, 'en_IN.UTF-8') + except locale.Error: + try: + locale.setlocale(locale.LC_CTYPE, 'en_IN') + except locale.Error as exc: + self.skipTest(str(exc)) + loc = locale.getlocale(locale.LC_CTYPE) + locale.setlocale(locale.LC_CTYPE, loc) + self.assertEqual(locale.getlocale(locale.LC_CTYPE)[0], 'en_IN') + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is diff --git a/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst b/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst new file mode 100644 index 000000000000000..ed342d0252d9d08 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst @@ -0,0 +1,4 @@ +:mod:`locale` now maps the ``en_IN`` locale alias to ``en_IN.UTF-8`` +instead of the obsolete X11 ``ISO8859-1`` codeset. This restores +``setlocale(getlocale())`` round-trips on modern glibc systems where +``en_IN`` is UTF-8-only. diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index f825862ffdf350a..6c1b2ffa8a40a7a 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -154,6 +154,9 @@ def check(data): # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist # on all platforms. data['c.utf8'] = 'C.UTF-8' + # Hardcode 'en_in' -> 'en_IN.UTF-8'. X11 locale.alias still maps en_IN to + # ISO8859-1 and would otherwise override glibc's en_IN/UTF-8 (gh-151316). + data['en_in'] = 'en_IN.UTF-8' while True: # Repeat optimization while the size is decreased. n = len(data) From a273b30e79b4b40f4102b5e840da79c2d0b39428 Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Wed, 12 Aug 2026 14:01:08 +0530 Subject: [PATCH 2/3] gh-151316: Prefer UTF-8 for glibc UTF-8-only locale aliases Sweep bare locales that are UTF-8-only in glibc SUPPORTED but still mapped to obsolete X11 codesets, restore those defaults in makelocalealias, and test the hardcoded replacements as a class. Co-authored-by: Cursor --- Lib/locale.py | 42 ++++----- Lib/test/test_locale.py | 85 +++++++++++-------- ...-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst | 9 +- Tools/i18n/makelocalealias.py | 17 +++- 4 files changed, 88 insertions(+), 65 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py index e37fe73ae039d93..03fc1dd844dd8b9 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -923,9 +923,9 @@ def getpreferredencoding(do_setlocale=True): # removed 'uz_uz@cyrillic' # # gh-151316: -# Prefer UTF-8 for en_IN. X11 locale.alias maps en_IN to ISO8859-1 and that -# entry overrides glibc's en_IN/UTF-8 during makelocalealias regeneration, -# so hardcode the modern codeset (mirroring the c.utf8 carve-out). +# Prefer glibc UTF-8 for bare locales that are UTF-8-only in SUPPORTED. +# X11 locale.alias still maps these to obsolete codesets and would otherwise +# override glibc during makelocalealias regeneration. locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -970,7 +970,7 @@ def getpreferredencoding(do_setlocale=True): 'ast_es': 'ast_ES.ISO8859-15', 'ayc_pe': 'ayc_PE.UTF-8', 'az': 'az_AZ.ISO8859-9E', - 'az_az': 'az_AZ.ISO8859-9E', + 'az_az': 'az_AZ.UTF-8', 'az_az.iso88599e': 'az_AZ.ISO8859-9E', 'az_ir': 'az_IR.UTF-8', 'be': 'be_BY.CP1251', @@ -1044,7 +1044,7 @@ def getpreferredencoding(do_setlocale=True): 'de_ch': 'de_CH.ISO8859-1', 'de_de': 'de_DE.ISO8859-1', 'de_it': 'de_IT.UTF-8', - 'de_li': 'de_LI.ISO8859-1', + 'de_li': 'de_LI.UTF-8', 'de_lu': 'de_LU.ISO8859-1', 'deutsch': 'de_DE.ISO8859-1', 'doi_in': 'doi_IN.UTF-8', @@ -1070,7 +1070,7 @@ def getpreferredencoding(do_setlocale=True): 'en_gb': 'en_GB.ISO8859-1', 'en_hk': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', - 'en_il': 'en_IL.ISO8859-1', + 'en_il': 'en_IL.UTF-8', 'en_in': 'en_IN.UTF-8', 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', @@ -1090,7 +1090,7 @@ def getpreferredencoding(do_setlocale=True): 'english_united-states': 'en_US.ISO8859-1', 'english_united-states.437': 'C', 'english_us': 'en_US.ISO8859-1', - 'eo': 'eo_XX.ISO8859-3', + 'eo': 'eo.UTF-8', 'eo.utf8': 'eo.UTF-8', 'eo_eo': 'eo_EO.ISO8859-3', 'eo_us.utf8': 'eo_US.UTF-8', @@ -1101,7 +1101,7 @@ def getpreferredencoding(do_setlocale=True): 'es_cl': 'es_CL.ISO8859-1', 'es_co': 'es_CO.ISO8859-1', 'es_cr': 'es_CR.ISO8859-1', - 'es_cu': 'es_CU.ISO8859-1', + 'es_cu': 'es_CU.UTF-8', 'es_do': 'es_DO.ISO8859-1', 'es_ec': 'es_EC.ISO8859-1', 'es_es': 'es_ES.ISO8859-1', @@ -1172,7 +1172,7 @@ def getpreferredencoding(do_setlocale=True): 'he_il': 'he_IL.ISO8859-8', 'hebrew': 'he_IL.ISO8859-8', 'hi': 'hi_IN.ISCII-DEV', - 'hi_in': 'hi_IN.ISCII-DEV', + 'hi_in': 'hi_IN.UTF-8', 'hi_in.isciidev': 'hi_IN.ISCII-DEV', 'hif_fj': 'hif_FJ.UTF-8', 'hne': 'hne_IN.UTF-8', @@ -1210,7 +1210,7 @@ def getpreferredencoding(do_setlocale=True): 'it_it': 'it_IT.ISO8859-1', 'italian': 'it_IT.ISO8859-1', 'iu': 'iu_CA.NUNACOM-8', - 'iu_ca': 'iu_CA.NUNACOM-8', + 'iu_ca': 'iu_CA.UTF-8', 'iu_ca.nunacom8': 'iu_CA.NUNACOM-8', 'iw': 'he_IL.ISO8859-8', 'iw_il': 'he_IL.ISO8859-8', @@ -1260,7 +1260,7 @@ def getpreferredencoding(do_setlocale=True): 'lithuanian': 'lt_LT.ISO8859-13', 'ln_cd': 'ln_CD.UTF-8', 'lo': 'lo_LA.MULELAO-1', - 'lo_la': 'lo_LA.MULELAO-1', + 'lo_la': 'lo_LA.UTF-8', 'lo_la.cp1133': 'lo_LA.IBM-CP1133', 'lo_la.ibmcp1133': 'lo_LA.IBM-CP1133', 'lo_la.mulelao1': 'lo_LA.MULELAO-1', @@ -1318,9 +1318,9 @@ def getpreferredencoding(do_setlocale=True): 'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1', 'norwegian': 'no_NO.ISO8859-1', 'nr': 'nr_ZA.ISO8859-1', - 'nr_za': 'nr_ZA.ISO8859-1', + 'nr_za': 'nr_ZA.UTF-8', 'nso': 'nso_ZA.ISO8859-15', - 'nso_za': 'nso_ZA.ISO8859-15', + 'nso_za': 'nso_ZA.UTF-8', 'ny': 'ny_NO.ISO8859-1', 'ny_no': 'ny_NO.ISO8859-1', 'nynorsk': 'nn_NO.ISO8859-1', @@ -1367,7 +1367,7 @@ def getpreferredencoding(do_setlocale=True): 'rumanian': 'ro_RO.ISO8859-2', 'russian': 'ru_RU.ISO8859-5', 'rw': 'rw_RW.ISO8859-1', - 'rw_rw': 'rw_RW.ISO8859-1', + 'rw_rw': 'rw_RW.UTF-8', 'sa_in': 'sa_IN.UTF-8', 'sah_ru': 'sah_RU.UTF-8', 'sat_in': 'sat_IN.UTF-8', @@ -1432,7 +1432,7 @@ def getpreferredencoding(do_setlocale=True): 'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8', 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', - 'ss_za': 'ss_ZA.ISO8859-1', + 'ss_za': 'ss_ZA.UTF-8', 'ssy_er': 'ssy_ER.UTF-8', 'st': 'st_ZA.ISO8859-1', 'st_za': 'st_ZA.ISO8859-1', @@ -1446,7 +1446,7 @@ def getpreferredencoding(do_setlocale=True): 'syr': 'syr.UTF-8', 'szl_pl': 'szl_PL.UTF-8', 'ta': 'ta_IN.TSCII-0', - 'ta_in': 'ta_IN.TSCII-0', + 'ta_in': 'ta_IN.UTF-8', 'ta_in.tscii': 'ta_IN.TSCII-0', 'ta_in.tscii0': 'ta_IN.TSCII-0', 'ta_lk': 'ta_LK.UTF-8', @@ -1468,7 +1468,7 @@ def getpreferredencoding(do_setlocale=True): 'tl': 'tl_PH.ISO8859-1', 'tl_ph': 'tl_PH.ISO8859-1', 'tn': 'tn_ZA.ISO8859-15', - 'tn_za': 'tn_ZA.ISO8859-15', + 'tn_za': 'tn_ZA.UTF-8', 'to_to': 'to_TO.UTF-8', 'tok': 'tok.UTF-8', 'tpi_pg': 'tpi_PG.UTF-8', @@ -1476,9 +1476,9 @@ def getpreferredencoding(do_setlocale=True): 'tr_cy': 'tr_CY.ISO8859-9', 'tr_tr': 'tr_TR.ISO8859-9', 'ts': 'ts_ZA.ISO8859-1', - 'ts_za': 'ts_ZA.ISO8859-1', + 'ts_za': 'ts_ZA.UTF-8', 'tt': 'tt_RU.TATAR-CYR', - 'tt_ru': 'tt_RU.TATAR-CYR', + 'tt_ru': 'tt_RU.UTF-8', 'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR', 'tt_ru@iqtelif': 'tt_RU.UTF-8@iqtelif', 'turkish': 'tr_TR.ISO8859-9', @@ -1490,13 +1490,13 @@ def getpreferredencoding(do_setlocale=True): 'unm_us': 'unm_US.UTF-8', 'ur': 'ur_PK.CP1256', 'ur_in': 'ur_IN.UTF-8', - 'ur_pk': 'ur_PK.CP1256', + 'ur_pk': 'ur_PK.UTF-8', 'uz': 'uz_UZ.UTF-8', 'uz_uz': 'uz_UZ.UTF-8', 've': 've_ZA.UTF-8', 've_za': 've_ZA.UTF-8', 'vi': 'vi_VN.TCVN', - 'vi_vn': 'vi_VN.TCVN', + 'vi_vn': 'vi_VN.UTF-8', 'vi_vn.tcvn': 'vi_VN.TCVN', 'vi_vn.tcvn5712': 'vi_VN.TCVN', 'vi_vn.viscii': 'vi_VN.VISCII', diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 3570f96fdfe7786..567d4853a9eca64 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -417,13 +417,6 @@ def test_english(self): self.check('english', 'en_EN.ISO8859-1') self.check('english_uk.ascii', 'en_GB.ISO8859-1') - def test_en_in_utf8(self): - # gh-151316: en_IN is UTF-8 on modern glibc; do not invent ISO8859-1. - self.check('en_IN', 'en_IN.UTF-8') - self.check('en_in', 'en_IN.UTF-8') - self.assertEqual(locale._parse_localename('en_IN'), ('en_IN', 'UTF-8')) - self.assertEqual(locale._parse_localename('en_in'), ('en_IN', 'UTF-8')) - def test_hyphenated_encoding(self): self.check('az_AZ.iso88599e', 'az_AZ.ISO8859-9E') self.check('az_AZ.ISO8859-9E', 'az_AZ.ISO8859-9E') @@ -649,36 +642,56 @@ def test_getlocale_with_modifier(self, localename, localetuple): self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple) -class TestEnINLocale(unittest.TestCase): - """gh-151316: en_IN must round-trip without inventing ISO8859-1.""" - - def setUp(self): - self.oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, self.oldlocale) - - def test_getlocale_setlocale_roundtrip(self): - try: - locale.setlocale(locale.LC_CTYPE, 'en_IN') - except locale.Error as exc: - self.skipTest(str(exc)) - loc = locale.getlocale(locale.LC_CTYPE) - self.assertEqual(loc[0], 'en_IN') - self.assertNotEqual(loc[1], 'ISO8859-1') - locale.setlocale(locale.LC_CTYPE, loc) - self.assertEqual(locale.getlocale(locale.LC_CTYPE), loc) +class TestHardcodedLocaleReplacements(unittest.TestCase): + """gh-151316: UTF-8 preferences that override X11 legacy codesets. + + These bare aliases are UTF-8-only in glibc SUPPORTED, but X11 + locale.alias still maps them to obsolete encodings. makelocalealias + restores the glibc UTF-8 defaults after applying X11 (plus de_li, + which has no bare SUPPORTED line). + """ + + REPLACEMENTS = { + 'az_az': 'az_AZ.UTF-8', + 'de_li': 'de_LI.UTF-8', + 'en_il': 'en_IL.UTF-8', + 'en_in': 'en_IN.UTF-8', + 'eo': 'eo.UTF-8', + 'es_cu': 'es_CU.UTF-8', + 'hi_in': 'hi_IN.UTF-8', + 'iu_ca': 'iu_CA.UTF-8', + 'lo_la': 'lo_LA.UTF-8', + 'nr_za': 'nr_ZA.UTF-8', + 'nso_za': 'nso_ZA.UTF-8', + 'rw_rw': 'rw_RW.UTF-8', + 'ss_za': 'ss_ZA.UTF-8', + 'ta_in': 'ta_IN.UTF-8', + 'tn_za': 'tn_ZA.UTF-8', + 'ts_za': 'ts_ZA.UTF-8', + 'tt_ru': 'tt_RU.UTF-8', + 'ur_pk': 'ur_PK.UTF-8', + 'vi_vn': 'vi_VN.UTF-8', + } - def test_setlocale_from_getlocale_tuple(self): - # Reproduces the issue report: setlocale(LC_*, getlocale()). - try: - locale.setlocale(locale.LC_CTYPE, 'en_IN.UTF-8') - except locale.Error: - try: - locale.setlocale(locale.LC_CTYPE, 'en_IN') - except locale.Error as exc: - self.skipTest(str(exc)) - loc = locale.getlocale(locale.LC_CTYPE) - locale.setlocale(locale.LC_CTYPE, loc) - self.assertEqual(locale.getlocale(locale.LC_CTYPE)[0], 'en_IN') + def test_locale_alias_entries(self): + for key, expected in self.REPLACEMENTS.items(): + with self.subTest(key=key): + self.assertEqual(locale.locale_alias[key], expected) + + def test_normalize(self): + for key, expected in self.REPLACEMENTS.items(): + with self.subTest(key=key): + self.assertEqual(locale.normalize(key), expected) + self.assertEqual(locale.normalize(key.upper()), expected) + + def test_parse_localename(self): + for key, expected in self.REPLACEMENTS.items(): + with self.subTest(key=key): + lang, encoding = expected.split('.') + self.assertEqual( + locale._parse_localename(key), + (lang, encoding), + ) class TestMiscellaneous(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst b/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst index ed342d0252d9d08..5223d7ff4d19954 100644 --- a/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst +++ b/Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst @@ -1,4 +1,5 @@ -:mod:`locale` now maps the ``en_IN`` locale alias to ``en_IN.UTF-8`` -instead of the obsolete X11 ``ISO8859-1`` codeset. This restores -``setlocale(getlocale())`` round-trips on modern glibc systems where -``en_IN`` is UTF-8-only. +:mod:`locale` now prefers UTF-8 for bare locale aliases that are +UTF-8-only in glibc (for example ``en_IN``, ``hi_IN``, ``es_CU``) +instead of obsolete X11 codesets. This restores +``setlocale(getlocale())`` round-trips on modern systems where those +locales have no legacy encoding. diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index 6c1b2ffa8a40a7a..364ae8d8f8e1a78 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -149,14 +149,23 @@ def check(data): args = parser.parse_args() data = locale.locale_alias.copy() - data.update(parse_glibc_supported(args.glibc_supported)) + glibc_data = parse_glibc_supported(args.glibc_supported) + data.update(glibc_data) data.update(parse(args.locale_alias)) # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist # on all platforms. data['c.utf8'] = 'C.UTF-8' - # Hardcode 'en_in' -> 'en_IN.UTF-8'. X11 locale.alias still maps en_IN to - # ISO8859-1 and would otherwise override glibc's en_IN/UTF-8 (gh-151316). - data['en_in'] = 'en_IN.UTF-8' + # Prefer glibc UTF-8 defaults over X11 legacy codesets (gh-151316). + # X11 locale.alias still maps several UTF-8-only locales to obsolete + # encodings and would otherwise override glibc during regeneration. + for key, value in glibc_data.items(): + if data.get(key) == value: + continue + if value.split('@')[0].endswith('.UTF-8'): + data[key] = value + # de_LI is UTF-8-only in glibc as 'de_LI.UTF-8', with no bare SUPPORTED + # line, so the X11 'de_LI.ISO8859-1' mapping would otherwise stick. + data['de_li'] = 'de_LI.UTF-8' while True: # Repeat optimization while the size is decreased. n = len(data) From 0f2ff4ca1e44e5cd42760e511235820dbc4f5fe4 Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Wed, 12 Aug 2026 14:35:49 +0530 Subject: [PATCH 3/3] gh-151316: Address review on locale UTF-8 overrides Use support.subTests for the hardcoded replacements, shorten the test docstring with an issue link, and extract apply_x11_locales_patch() so the X11 override of glibc UTF-8 defaults is explicit. Co-authored-by: Cursor --- Lib/test/test_locale.py | 85 +++++++++++++++++------------------ Tools/i18n/makelocalealias.py | 31 ++++++++----- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 567d4853a9eca64..9a929bc6d77a441 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -642,56 +642,51 @@ def test_getlocale_with_modifier(self, localename, localetuple): self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple) +# Bare aliases that must stay on UTF-8 after X11 locale.alias regeneration +# (gh-151316). Keep in sync with apply_x11_locales_patch() in +# Tools/i18n/makelocalealias.py. +_UTF8_X11_OVERRIDES = ( + ('az_az', 'az_AZ.UTF-8'), + ('de_li', 'de_LI.UTF-8'), + ('en_il', 'en_IL.UTF-8'), + ('en_in', 'en_IN.UTF-8'), + ('eo', 'eo.UTF-8'), + ('es_cu', 'es_CU.UTF-8'), + ('hi_in', 'hi_IN.UTF-8'), + ('iu_ca', 'iu_CA.UTF-8'), + ('lo_la', 'lo_LA.UTF-8'), + ('nr_za', 'nr_ZA.UTF-8'), + ('nso_za', 'nso_ZA.UTF-8'), + ('rw_rw', 'rw_RW.UTF-8'), + ('ss_za', 'ss_ZA.UTF-8'), + ('ta_in', 'ta_IN.UTF-8'), + ('tn_za', 'tn_ZA.UTF-8'), + ('ts_za', 'ts_ZA.UTF-8'), + ('tt_ru', 'tt_RU.UTF-8'), + ('ur_pk', 'ur_PK.UTF-8'), + ('vi_vn', 'vi_VN.UTF-8'), +) + + class TestHardcodedLocaleReplacements(unittest.TestCase): - """gh-151316: UTF-8 preferences that override X11 legacy codesets. + """UTF-8 preferences that override X11 legacy codesets. - These bare aliases are UTF-8-only in glibc SUPPORTED, but X11 - locale.alias still maps them to obsolete encodings. makelocalealias - restores the glibc UTF-8 defaults after applying X11 (plus de_li, - which has no bare SUPPORTED line). + See https://github.com/python/cpython/issues/151316 """ - REPLACEMENTS = { - 'az_az': 'az_AZ.UTF-8', - 'de_li': 'de_LI.UTF-8', - 'en_il': 'en_IL.UTF-8', - 'en_in': 'en_IN.UTF-8', - 'eo': 'eo.UTF-8', - 'es_cu': 'es_CU.UTF-8', - 'hi_in': 'hi_IN.UTF-8', - 'iu_ca': 'iu_CA.UTF-8', - 'lo_la': 'lo_LA.UTF-8', - 'nr_za': 'nr_ZA.UTF-8', - 'nso_za': 'nso_ZA.UTF-8', - 'rw_rw': 'rw_RW.UTF-8', - 'ss_za': 'ss_ZA.UTF-8', - 'ta_in': 'ta_IN.UTF-8', - 'tn_za': 'tn_ZA.UTF-8', - 'ts_za': 'ts_ZA.UTF-8', - 'tt_ru': 'tt_RU.UTF-8', - 'ur_pk': 'ur_PK.UTF-8', - 'vi_vn': 'vi_VN.UTF-8', - } + @support.subTests('key,expected', _UTF8_X11_OVERRIDES) + def test_locale_alias_entries(self, key, expected): + self.assertEqual(locale.locale_alias[key], expected) + + @support.subTests('key,expected', _UTF8_X11_OVERRIDES) + def test_normalize(self, key, expected): + self.assertEqual(locale.normalize(key), expected) + self.assertEqual(locale.normalize(key.upper()), expected) - def test_locale_alias_entries(self): - for key, expected in self.REPLACEMENTS.items(): - with self.subTest(key=key): - self.assertEqual(locale.locale_alias[key], expected) - - def test_normalize(self): - for key, expected in self.REPLACEMENTS.items(): - with self.subTest(key=key): - self.assertEqual(locale.normalize(key), expected) - self.assertEqual(locale.normalize(key.upper()), expected) - - def test_parse_localename(self): - for key, expected in self.REPLACEMENTS.items(): - with self.subTest(key=key): - lang, encoding = expected.split('.') - self.assertEqual( - locale._parse_localename(key), - (lang, encoding), - ) + @support.subTests('key,expected', _UTF8_X11_OVERRIDES) + def test_parse_localename(self, key, expected): + lang, encoding = expected.split('.') + self.assertEqual(locale._parse_localename(key), (lang, encoding)) class TestMiscellaneous(unittest.TestCase): diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index 364ae8d8f8e1a78..99442a3d47f4dd6 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -98,6 +98,25 @@ def parse_glibc_supported(filename): data[locale] = alias return data +def apply_x11_locales_patch(data, glibc_data): + """Restore glibc UTF-8 defaults that X11 locale.alias overwrote. + + Regeneration updates glibc SUPPORTED first, then X11 locale.alias. + X11 still maps several locales that are UTF-8-only in modern glibc to + obsolete encodings (for example en_IN -> en_IN.ISO8859-1). Prefer the + glibc UTF-8 mapping for those bare keys (gh-151316). + + de_LI is special: glibc only ships de_LI.UTF-8 (no bare SUPPORTED line), + so the X11 de_LI.ISO8859-1 mapping would otherwise stick. + """ + for key, value in glibc_data.items(): + if data.get(key) == value: + continue + if value.split('@')[0].endswith('.UTF-8'): + data[key] = value + data['de_li'] = 'de_LI.UTF-8' + return data + def pprint(data): items = sorted(data.items()) for k, v in items: @@ -155,17 +174,7 @@ def check(data): # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist # on all platforms. data['c.utf8'] = 'C.UTF-8' - # Prefer glibc UTF-8 defaults over X11 legacy codesets (gh-151316). - # X11 locale.alias still maps several UTF-8-only locales to obsolete - # encodings and would otherwise override glibc during regeneration. - for key, value in glibc_data.items(): - if data.get(key) == value: - continue - if value.split('@')[0].endswith('.UTF-8'): - data[key] = value - # de_LI is UTF-8-only in glibc as 'de_LI.UTF-8', with no bare SUPPORTED - # line, so the X11 'de_LI.ISO8859-1' mapping would otherwise stick. - data['de_li'] = 'de_LI.UTF-8' + data = apply_x11_locales_patch(data, glibc_data) while True: # Repeat optimization while the size is decreased. n = len(data)