Skip to content

Commit a273b30

Browse files
sankalpsthakurcursoragent
sankalpsthakur
and
cursoragent
committed
gh-151316: Prefer UTF-8 for glibc UTF-8-only locale aliases
Sweep bare locales that are UTF-8-only in glibc SUPPORTED but still mapped to obsolete X11 codesets, restore those defaults in makelocalealias, and test the hardcoded replacements as a class. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent b959df9 commit a273b30

4 files changed

Lines changed: 88 additions & 65 deletions

File tree

‎Lib/locale.py‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -923,9 +923,9 @@ def getpreferredencoding(do_setlocale=True):
923923
# removed 'uz_uz@cyrillic'
924924
#
925925
# gh-151316:
926-
# Prefer UTF-8 for en_IN. X11 locale.alias maps en_IN to ISO8859-1 and that
927-
# entry overrides glibc's en_IN/UTF-8 during makelocalealias regeneration,
928-
# so hardcode the modern codeset (mirroring the c.utf8 carve-out).
926+
# Prefer glibc UTF-8 for bare locales that are UTF-8-only in SUPPORTED.
927+
# X11 locale.alias still maps these to obsolete codesets and would otherwise
928+
# override glibc during makelocalealias regeneration.
929929

930930
locale_alias = {
931931
'a3': 'az_AZ.KOI8-C',
@@ -970,7 +970,7 @@ def getpreferredencoding(do_setlocale=True):
970970
'ast_es': 'ast_ES.ISO8859-15',
971971
'ayc_pe': 'ayc_PE.UTF-8',
972972
'az': 'az_AZ.ISO8859-9E',
973-
'az_az': 'az_AZ.ISO8859-9E',
973+
'az_az': 'az_AZ.UTF-8',
974974
'az_az.iso88599e': 'az_AZ.ISO8859-9E',
975975
'az_ir': 'az_IR.UTF-8',
976976
'be': 'be_BY.CP1251',
@@ -1044,7 +1044,7 @@ def getpreferredencoding(do_setlocale=True):
10441044
'de_ch': 'de_CH.ISO8859-1',
10451045
'de_de': 'de_DE.ISO8859-1',
10461046
'de_it': 'de_IT.UTF-8',
1047-
'de_li': 'de_LI.ISO8859-1',
1047+
'de_li': 'de_LI.UTF-8',
10481048
'de_lu': 'de_LU.ISO8859-1',
10491049
'deutsch': 'de_DE.ISO8859-1',
10501050
'doi_in': 'doi_IN.UTF-8',
@@ -1070,7 +1070,7 @@ def getpreferredencoding(do_setlocale=True):
10701070
'en_gb': 'en_GB.ISO8859-1',
10711071
'en_hk': 'en_HK.ISO8859-1',
10721072
'en_ie': 'en_IE.ISO8859-1',
1073-
'en_il': 'en_IL.ISO8859-1',
1073+
'en_il': 'en_IL.UTF-8',
10741074
'en_in': 'en_IN.UTF-8',
10751075
'en_ng': 'en_NG.UTF-8',
10761076
'en_nz': 'en_NZ.ISO8859-1',
@@ -1090,7 +1090,7 @@ def getpreferredencoding(do_setlocale=True):
10901090
'english_united-states': 'en_US.ISO8859-1',
10911091
'english_united-states.437': 'C',
10921092
'english_us': 'en_US.ISO8859-1',
1093-
'eo': 'eo_XX.ISO8859-3',
1093+
'eo': 'eo.UTF-8',
10941094
'eo.utf8': 'eo.UTF-8',
10951095
'eo_eo': 'eo_EO.ISO8859-3',
10961096
'eo_us.utf8': 'eo_US.UTF-8',
@@ -1101,7 +1101,7 @@ def getpreferredencoding(do_setlocale=True):
11011101
'es_cl': 'es_CL.ISO8859-1',
11021102
'es_co': 'es_CO.ISO8859-1',
11031103
'es_cr': 'es_CR.ISO8859-1',
1104-
'es_cu': 'es_CU.ISO8859-1',
1104+
'es_cu': 'es_CU.UTF-8',
11051105
'es_do': 'es_DO.ISO8859-1',
11061106
'es_ec': 'es_EC.ISO8859-1',
11071107
'es_es': 'es_ES.ISO8859-1',
@@ -1172,7 +1172,7 @@ def getpreferredencoding(do_setlocale=True):
11721172
'he_il': 'he_IL.ISO8859-8',
11731173
'hebrew': 'he_IL.ISO8859-8',
11741174
'hi': 'hi_IN.ISCII-DEV',
1175-
'hi_in': 'hi_IN.ISCII-DEV',
1175+
'hi_in': 'hi_IN.UTF-8',
11761176
'hi_in.isciidev': 'hi_IN.ISCII-DEV',
11771177
'hif_fj': 'hif_FJ.UTF-8',
11781178
'hne': 'hne_IN.UTF-8',
@@ -1210,7 +1210,7 @@ def getpreferredencoding(do_setlocale=True):
12101210
'it_it': 'it_IT.ISO8859-1',
12111211
'italian': 'it_IT.ISO8859-1',
12121212
'iu': 'iu_CA.NUNACOM-8',
1213-
'iu_ca': 'iu_CA.NUNACOM-8',
1213+
'iu_ca': 'iu_CA.UTF-8',
12141214
'iu_ca.nunacom8': 'iu_CA.NUNACOM-8',
12151215
'iw': 'he_IL.ISO8859-8',
12161216
'iw_il': 'he_IL.ISO8859-8',
@@ -1260,7 +1260,7 @@ def getpreferredencoding(do_setlocale=True):
12601260
'lithuanian': 'lt_LT.ISO8859-13',
12611261
'ln_cd': 'ln_CD.UTF-8',
12621262
'lo': 'lo_LA.MULELAO-1',
1263-
'lo_la': 'lo_LA.MULELAO-1',
1263+
'lo_la': 'lo_LA.UTF-8',
12641264
'lo_la.cp1133': 'lo_LA.IBM-CP1133',
12651265
'lo_la.ibmcp1133': 'lo_LA.IBM-CP1133',
12661266
'lo_la.mulelao1': 'lo_LA.MULELAO-1',
@@ -1318,9 +1318,9 @@ def getpreferredencoding(do_setlocale=True):
13181318
'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1',
13191319
'norwegian': 'no_NO.ISO8859-1',
13201320
'nr': 'nr_ZA.ISO8859-1',
1321-
'nr_za': 'nr_ZA.ISO8859-1',
1321+
'nr_za': 'nr_ZA.UTF-8',
13221322
'nso': 'nso_ZA.ISO8859-15',
1323-
'nso_za': 'nso_ZA.ISO8859-15',
1323+
'nso_za': 'nso_ZA.UTF-8',
13241324
'ny': 'ny_NO.ISO8859-1',
13251325
'ny_no': 'ny_NO.ISO8859-1',
13261326
'nynorsk': 'nn_NO.ISO8859-1',
@@ -1367,7 +1367,7 @@ def getpreferredencoding(do_setlocale=True):
13671367
'rumanian': 'ro_RO.ISO8859-2',
13681368
'russian': 'ru_RU.ISO8859-5',
13691369
'rw': 'rw_RW.ISO8859-1',
1370-
'rw_rw': 'rw_RW.ISO8859-1',
1370+
'rw_rw': 'rw_RW.UTF-8',
13711371
'sa_in': 'sa_IN.UTF-8',
13721372
'sah_ru': 'sah_RU.UTF-8',
13731373
'sat_in': 'sat_IN.UTF-8',
@@ -1432,7 +1432,7 @@ def getpreferredencoding(do_setlocale=True):
14321432
'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8',
14331433
'sr_yu@cyrillic': 'sr_RS.UTF-8',
14341434
'ss': 'ss_ZA.ISO8859-1',
1435-
'ss_za': 'ss_ZA.ISO8859-1',
1435+
'ss_za': 'ss_ZA.UTF-8',
14361436
'ssy_er': 'ssy_ER.UTF-8',
14371437
'st': 'st_ZA.ISO8859-1',
14381438
'st_za': 'st_ZA.ISO8859-1',
@@ -1446,7 +1446,7 @@ def getpreferredencoding(do_setlocale=True):
14461446
'syr': 'syr.UTF-8',
14471447
'szl_pl': 'szl_PL.UTF-8',
14481448
'ta': 'ta_IN.TSCII-0',
1449-
'ta_in': 'ta_IN.TSCII-0',
1449+
'ta_in': 'ta_IN.UTF-8',
14501450
'ta_in.tscii': 'ta_IN.TSCII-0',
14511451
'ta_in.tscii0': 'ta_IN.TSCII-0',
14521452
'ta_lk': 'ta_LK.UTF-8',
@@ -1468,17 +1468,17 @@ def getpreferredencoding(do_setlocale=True):
14681468
'tl': 'tl_PH.ISO8859-1',
14691469
'tl_ph': 'tl_PH.ISO8859-1',
14701470
'tn': 'tn_ZA.ISO8859-15',
1471-
'tn_za': 'tn_ZA.ISO8859-15',
1471+
'tn_za': 'tn_ZA.UTF-8',
14721472
'to_to': 'to_TO.UTF-8',
14731473
'tok': 'tok.UTF-8',
14741474
'tpi_pg': 'tpi_PG.UTF-8',
14751475
'tr': 'tr_TR.ISO8859-9',
14761476
'tr_cy': 'tr_CY.ISO8859-9',
14771477
'tr_tr': 'tr_TR.ISO8859-9',
14781478
'ts': 'ts_ZA.ISO8859-1',
1479-
'ts_za': 'ts_ZA.ISO8859-1',
1479+
'ts_za': 'ts_ZA.UTF-8',
14801480
'tt': 'tt_RU.TATAR-CYR',
1481-
'tt_ru': 'tt_RU.TATAR-CYR',
1481+
'tt_ru': 'tt_RU.UTF-8',
14821482
'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR',
14831483
'tt_ru@iqtelif': 'tt_RU.UTF-8@iqtelif',
14841484
'turkish': 'tr_TR.ISO8859-9',
@@ -1490,13 +1490,13 @@ def getpreferredencoding(do_setlocale=True):
14901490
'unm_us': 'unm_US.UTF-8',
14911491
'ur': 'ur_PK.CP1256',
14921492
'ur_in': 'ur_IN.UTF-8',
1493-
'ur_pk': 'ur_PK.CP1256',
1493+
'ur_pk': 'ur_PK.UTF-8',
14941494
'uz': 'uz_UZ.UTF-8',
14951495
'uz_uz': 'uz_UZ.UTF-8',
14961496
've': 've_ZA.UTF-8',
14971497
've_za': 've_ZA.UTF-8',
14981498
'vi': 'vi_VN.TCVN',
1499-
'vi_vn': 'vi_VN.TCVN',
1499+
'vi_vn': 'vi_VN.UTF-8',
15001500
'vi_vn.tcvn': 'vi_VN.TCVN',
15011501
'vi_vn.tcvn5712': 'vi_VN.TCVN',
15021502
'vi_vn.viscii': 'vi_VN.VISCII',

‎Lib/test/test_locale.py‎

Lines changed: 49 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -417,13 +417,6 @@ def test_english(self):
417417
self.check('english', 'en_EN.ISO8859-1')
418418
self.check('english_uk.ascii', 'en_GB.ISO8859-1')
419419

420-
def test_en_in_utf8(self):
421-
# gh-151316: en_IN is UTF-8 on modern glibc; do not invent ISO8859-1.
422-
self.check('en_IN', 'en_IN.UTF-8')
423-
self.check('en_in', 'en_IN.UTF-8')
424-
self.assertEqual(locale._parse_localename('en_IN'), ('en_IN', 'UTF-8'))
425-
self.assertEqual(locale._parse_localename('en_in'), ('en_IN', 'UTF-8'))
426-
427420
def test_hyphenated_encoding(self):
428421
self.check('az_AZ.iso88599e', 'az_AZ.ISO8859-9E')
429422
self.check('az_AZ.ISO8859-9E', 'az_AZ.ISO8859-9E')
@@ -649,36 +642,56 @@ def test_getlocale_with_modifier(self, localename, localetuple):
649642
self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple)
650643

651644

652-
class TestEnINLocale(unittest.TestCase):
653-
"""gh-151316: en_IN must round-trip without inventing ISO8859-1."""
654-
655-
def setUp(self):
656-
self.oldlocale = locale.setlocale(locale.LC_CTYPE)
657-
self.addCleanup(locale.setlocale, locale.LC_CTYPE, self.oldlocale)
658-
659-
def test_getlocale_setlocale_roundtrip(self):
660-
try:
661-
locale.setlocale(locale.LC_CTYPE, 'en_IN')
662-
except locale.Error as exc:
663-
self.skipTest(str(exc))
664-
loc = locale.getlocale(locale.LC_CTYPE)
665-
self.assertEqual(loc[0], 'en_IN')
666-
self.assertNotEqual(loc[1], 'ISO8859-1')
667-
locale.setlocale(locale.LC_CTYPE, loc)
668-
self.assertEqual(locale.getlocale(locale.LC_CTYPE), loc)
645+
class TestHardcodedLocaleReplacements(unittest.TestCase):
646+
"""gh-151316: UTF-8 preferences that override X11 legacy codesets.
647+
648+
These bare aliases are UTF-8-only in glibc SUPPORTED, but X11
649+
locale.alias still maps them to obsolete encodings. makelocalealias
650+
restores the glibc UTF-8 defaults after applying X11 (plus de_li,
651+
which has no bare SUPPORTED line).
652+
"""
653+
654+
REPLACEMENTS = {
655+
'az_az': 'az_AZ.UTF-8',
656+
'de_li': 'de_LI.UTF-8',
657+
'en_il': 'en_IL.UTF-8',
658+
'en_in': 'en_IN.UTF-8',
659+
'eo': 'eo.UTF-8',
660+
'es_cu': 'es_CU.UTF-8',
661+
'hi_in': 'hi_IN.UTF-8',
662+
'iu_ca': 'iu_CA.UTF-8',
663+
'lo_la': 'lo_LA.UTF-8',
664+
'nr_za': 'nr_ZA.UTF-8',
665+
'nso_za': 'nso_ZA.UTF-8',
666+
'rw_rw': 'rw_RW.UTF-8',
667+
'ss_za': 'ss_ZA.UTF-8',
668+
'ta_in': 'ta_IN.UTF-8',
669+
'tn_za': 'tn_ZA.UTF-8',
670+
'ts_za': 'ts_ZA.UTF-8',
671+
'tt_ru': 'tt_RU.UTF-8',
672+
'ur_pk': 'ur_PK.UTF-8',
673+
'vi_vn': 'vi_VN.UTF-8',
674+
}
669675

670-
def test_setlocale_from_getlocale_tuple(self):
671-
# Reproduces the issue report: setlocale(LC_*, getlocale()).
672-
try:
673-
locale.setlocale(locale.LC_CTYPE, 'en_IN.UTF-8')
674-
except locale.Error:
675-
try:
676-
locale.setlocale(locale.LC_CTYPE, 'en_IN')
677-
except locale.Error as exc:
678-
self.skipTest(str(exc))
679-
loc = locale.getlocale(locale.LC_CTYPE)
680-
locale.setlocale(locale.LC_CTYPE, loc)
681-
self.assertEqual(locale.getlocale(locale.LC_CTYPE)[0], 'en_IN')
676+
def test_locale_alias_entries(self):
677+
for key, expected in self.REPLACEMENTS.items():
678+
with self.subTest(key=key):
679+
self.assertEqual(locale.locale_alias[key], expected)
680+
681+
def test_normalize(self):
682+
for key, expected in self.REPLACEMENTS.items():
683+
with self.subTest(key=key):
684+
self.assertEqual(locale.normalize(key), expected)
685+
self.assertEqual(locale.normalize(key.upper()), expected)
686+
687+
def test_parse_localename(self):
688+
for key, expected in self.REPLACEMENTS.items():
689+
with self.subTest(key=key):
690+
lang, encoding = expected.split('.')
691+
self.assertEqual(
692+
locale._parse_localename(key),
693+
(lang, encoding),
694+
)
682695

683696

684697
class TestMiscellaneous(unittest.TestCase):

‎Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
:mod:`locale` now maps the ``en_IN`` locale alias to ``en_IN.UTF-8``
2-
instead of the obsolete X11 ``ISO8859-1`` codeset. This restores
3-
``setlocale(getlocale())`` round-trips on modern glibc systems where
4-
``en_IN`` is UTF-8-only.
1+
:mod:`locale` now prefers UTF-8 for bare locale aliases that are
2+
UTF-8-only in glibc (for example ``en_IN``, ``hi_IN``, ``es_CU``)
3+
instead of obsolete X11 codesets. This restores
4+
``setlocale(getlocale())`` round-trips on modern systems where those
5+
locales have no legacy encoding.

‎Tools/i18n/makelocalealias.py‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,14 +149,23 @@ def check(data):
149149
args = parser.parse_args()
150150

151151
data = locale.locale_alias.copy()
152-
data.update(parse_glibc_supported(args.glibc_supported))
152+
glibc_data = parse_glibc_supported(args.glibc_supported)
153+
data.update(glibc_data)
153154
data.update(parse(args.locale_alias))
154155
# Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist
155156
# on all platforms.
156157
data['c.utf8'] = 'C.UTF-8'
157-
# Hardcode 'en_in' -> 'en_IN.UTF-8'. X11 locale.alias still maps en_IN to
158-
# ISO8859-1 and would otherwise override glibc's en_IN/UTF-8 (gh-151316).
159-
data['en_in'] = 'en_IN.UTF-8'
158+
# Prefer glibc UTF-8 defaults over X11 legacy codesets (gh-151316).
159+
# X11 locale.alias still maps several UTF-8-only locales to obsolete
160+
# encodings and would otherwise override glibc during regeneration.
161+
for key, value in glibc_data.items():
162+
if data.get(key) == value:
163+
continue
164+
if value.split('@')[0].endswith('.UTF-8'):
165+
data[key] = value
166+
# de_LI is UTF-8-only in glibc as 'de_LI.UTF-8', with no bare SUPPORTED
167+
# line, so the X11 'de_LI.ISO8859-1' mapping would otherwise stick.
168+
data['de_li'] = 'de_LI.UTF-8'
160169
while True:
161170
# Repeat optimization while the size is decreased.
162171
n = len(data)

0 commit comments

Comments
 (0)