Skip to content

gh-151316: Prefer UTF-8 for glibc UTF-8-only locale aliases #155549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sankalpsthakur wants to merge 3 commits into from
+98 −20
Closed

gh-151316: Prefer UTF-8 for glibc UTF-8-only locale aliases #155549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b959df9
gh-151316: Prefer UTF-8 for the en_IN locale alias
sankalpsthakur Aug 11, 2026
a273b30
gh-151316: Prefer UTF-8 for glibc UTF-8-only locale aliases
sankalpsthakur Aug 12, 2026
0f2ff4c
gh-151316: Address review on locale UTF-8 overrides
sankalpsthakur Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 24 additions & 19 deletions Lib/locale.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -921,6 +921,11 @@ def getpreferredencoding(do_setlocale=True):
#
# removed 'el_gr@euro'
# removed 'uz_uz@cyrillic'
#
# gh-151316:
# Prefer glibc UTF-8 for bare locales that are UTF-8-only in SUPPORTED.
# X11 locale.alias still maps these to obsolete codesets and would otherwise
# override glibc during makelocalealias regeneration.

locale_alias = {
'a3': 'az_AZ.KOI8-C',
Expand Down Expand Up @@ -965,7 +970,7 @@ def getpreferredencoding(do_setlocale=True):
'ast_es': 'ast_ES.ISO8859-15',
'ayc_pe': 'ayc_PE.UTF-8',
'az': 'az_AZ.ISO8859-9E',
'az_az': 'az_AZ.ISO8859-9E',
'az_az': 'az_AZ.UTF-8',
'az_az.iso88599e': 'az_AZ.ISO8859-9E',
'az_ir': 'az_IR.UTF-8',
'be': 'be_BY.CP1251',
Expand Down Expand Up @@ -1039,7 +1044,7 @@ def getpreferredencoding(do_setlocale=True):
'de_ch': 'de_CH.ISO8859-1',
'de_de': 'de_DE.ISO8859-1',
'de_it': 'de_IT.UTF-8',
'de_li': 'de_LI.ISO8859-1',
'de_li': 'de_LI.UTF-8',
'de_lu': 'de_LU.ISO8859-1',
'deutsch': 'de_DE.ISO8859-1',
'doi_in': 'doi_IN.UTF-8',
Expand All @@ -1065,8 +1070,8 @@ def getpreferredencoding(do_setlocale=True):
'en_gb': 'en_GB.ISO8859-1',
'en_hk': 'en_HK.ISO8859-1',
'en_ie': 'en_IE.ISO8859-1',
'en_il': 'en_IL.ISO8859-1',
'en_in': 'en_IN.ISO8859-1',
'en_il': 'en_IL.UTF-8',
'en_in': 'en_IN.UTF-8',
'en_ng': 'en_NG.UTF-8',
'en_nz': 'en_NZ.ISO8859-1',
'en_ph': 'en_PH.ISO8859-1',
Expand All @@ -1085,7 +1090,7 @@ def getpreferredencoding(do_setlocale=True):
'english_united-states': 'en_US.ISO8859-1',
'english_united-states.437': 'C',
'english_us': 'en_US.ISO8859-1',
'eo': 'eo_XX.ISO8859-3',
'eo': 'eo.UTF-8',
'eo.utf8': 'eo.UTF-8',
'eo_eo': 'eo_EO.ISO8859-3',
'eo_us.utf8': 'eo_US.UTF-8',
Expand All @@ -1096,7 +1101,7 @@ def getpreferredencoding(do_setlocale=True):
'es_cl': 'es_CL.ISO8859-1',
'es_co': 'es_CO.ISO8859-1',
'es_cr': 'es_CR.ISO8859-1',
'es_cu': 'es_CU.ISO8859-1',
'es_cu': 'es_CU.UTF-8',
'es_do': 'es_DO.ISO8859-1',
'es_ec': 'es_EC.ISO8859-1',
'es_es': 'es_ES.ISO8859-1',
Expand Down Expand Up @@ -1167,7 +1172,7 @@ def getpreferredencoding(do_setlocale=True):
'he_il': 'he_IL.ISO8859-8',
'hebrew': 'he_IL.ISO8859-8',
'hi': 'hi_IN.ISCII-DEV',
'hi_in': 'hi_IN.ISCII-DEV',
'hi_in': 'hi_IN.UTF-8',
'hi_in.isciidev': 'hi_IN.ISCII-DEV',
'hif_fj': 'hif_FJ.UTF-8',
'hne': 'hne_IN.UTF-8',
Expand Down Expand Up @@ -1205,7 +1210,7 @@ def getpreferredencoding(do_setlocale=True):
'it_it': 'it_IT.ISO8859-1',
'italian': 'it_IT.ISO8859-1',
'iu': 'iu_CA.NUNACOM-8',
'iu_ca': 'iu_CA.NUNACOM-8',
'iu_ca': 'iu_CA.UTF-8',
'iu_ca.nunacom8': 'iu_CA.NUNACOM-8',
'iw': 'he_IL.ISO8859-8',
'iw_il': 'he_IL.ISO8859-8',
Expand Down Expand Up @@ -1255,7 +1260,7 @@ def getpreferredencoding(do_setlocale=True):
'lithuanian': 'lt_LT.ISO8859-13',
'ln_cd': 'ln_CD.UTF-8',
'lo': 'lo_LA.MULELAO-1',
'lo_la': 'lo_LA.MULELAO-1',
'lo_la': 'lo_LA.UTF-8',
'lo_la.cp1133': 'lo_LA.IBM-CP1133',
'lo_la.ibmcp1133': 'lo_LA.IBM-CP1133',
'lo_la.mulelao1': 'lo_LA.MULELAO-1',
Expand Down Expand Up @@ -1313,9 +1318,9 @@ def getpreferredencoding(do_setlocale=True):
'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1',
'norwegian': 'no_NO.ISO8859-1',
'nr': 'nr_ZA.ISO8859-1',
'nr_za': 'nr_ZA.ISO8859-1',
'nr_za': 'nr_ZA.UTF-8',
'nso': 'nso_ZA.ISO8859-15',
'nso_za': 'nso_ZA.ISO8859-15',
'nso_za': 'nso_ZA.UTF-8',
'ny': 'ny_NO.ISO8859-1',
'ny_no': 'ny_NO.ISO8859-1',
'nynorsk': 'nn_NO.ISO8859-1',
Expand Down Expand Up @@ -1362,7 +1367,7 @@ def getpreferredencoding(do_setlocale=True):
'rumanian': 'ro_RO.ISO8859-2',
'russian': 'ru_RU.ISO8859-5',
'rw': 'rw_RW.ISO8859-1',
'rw_rw': 'rw_RW.ISO8859-1',
'rw_rw': 'rw_RW.UTF-8',
'sa_in': 'sa_IN.UTF-8',
'sah_ru': 'sah_RU.UTF-8',
'sat_in': 'sat_IN.UTF-8',
Expand Down Expand Up @@ -1427,7 +1432,7 @@ def getpreferredencoding(do_setlocale=True):
'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8',
'sr_yu@cyrillic': 'sr_RS.UTF-8',
'ss': 'ss_ZA.ISO8859-1',
'ss_za': 'ss_ZA.ISO8859-1',
'ss_za': 'ss_ZA.UTF-8',
'ssy_er': 'ssy_ER.UTF-8',
'st': 'st_ZA.ISO8859-1',
'st_za': 'st_ZA.ISO8859-1',
Expand All @@ -1441,7 +1446,7 @@ def getpreferredencoding(do_setlocale=True):
'syr': 'syr.UTF-8',
'szl_pl': 'szl_PL.UTF-8',
'ta': 'ta_IN.TSCII-0',
'ta_in': 'ta_IN.TSCII-0',
'ta_in': 'ta_IN.UTF-8',
'ta_in.tscii': 'ta_IN.TSCII-0',
'ta_in.tscii0': 'ta_IN.TSCII-0',
'ta_lk': 'ta_LK.UTF-8',
Expand All @@ -1463,17 +1468,17 @@ def getpreferredencoding(do_setlocale=True):
'tl': 'tl_PH.ISO8859-1',
'tl_ph': 'tl_PH.ISO8859-1',
'tn': 'tn_ZA.ISO8859-15',
'tn_za': 'tn_ZA.ISO8859-15',
'tn_za': 'tn_ZA.UTF-8',
'to_to': 'to_TO.UTF-8',
'tok': 'tok.UTF-8',
'tpi_pg': 'tpi_PG.UTF-8',
'tr': 'tr_TR.ISO8859-9',
'tr_cy': 'tr_CY.ISO8859-9',
'tr_tr': 'tr_TR.ISO8859-9',
'ts': 'ts_ZA.ISO8859-1',
'ts_za': 'ts_ZA.ISO8859-1',
'ts_za': 'ts_ZA.UTF-8',
'tt': 'tt_RU.TATAR-CYR',
'tt_ru': 'tt_RU.TATAR-CYR',
'tt_ru': 'tt_RU.UTF-8',
'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR',
'tt_ru@iqtelif': 'tt_RU.UTF-8@iqtelif',
'turkish': 'tr_TR.ISO8859-9',
Expand All @@ -1485,13 +1490,13 @@ def getpreferredencoding(do_setlocale=True):
'unm_us': 'unm_US.UTF-8',
'ur': 'ur_PK.CP1256',
'ur_in': 'ur_IN.UTF-8',
'ur_pk': 'ur_PK.CP1256',
'ur_pk': 'ur_PK.UTF-8',
'uz': 'uz_UZ.UTF-8',
'uz_uz': 'uz_UZ.UTF-8',
've': 've_ZA.UTF-8',
've_za': 've_ZA.UTF-8',
'vi': 'vi_VN.TCVN',
'vi_vn': 'vi_VN.TCVN',
'vi_vn': 'vi_VN.UTF-8',
'vi_vn.tcvn': 'vi_VN.TCVN',
'vi_vn.tcvn5712': 'vi_VN.TCVN',
'vi_vn.viscii': 'vi_VN.VISCII',
Expand Down
47 changes: 47 additions & 0 deletions Lib/test/test_locale.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,53 @@ def test_getlocale_with_modifier(self, localename, localetuple):
self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple)


# Bare aliases that must stay on UTF-8 after X11 locale.alias regeneration
# (gh-151316). Keep in sync with apply_x11_locales_patch() in
# Tools/i18n/makelocalealias.py.
_UTF8_X11_OVERRIDES = (
('az_az', 'az_AZ.UTF-8'),
('de_li', 'de_LI.UTF-8'),
('en_il', 'en_IL.UTF-8'),
('en_in', 'en_IN.UTF-8'),
('eo', 'eo.UTF-8'),
('es_cu', 'es_CU.UTF-8'),
('hi_in', 'hi_IN.UTF-8'),
('iu_ca', 'iu_CA.UTF-8'),
('lo_la', 'lo_LA.UTF-8'),
('nr_za', 'nr_ZA.UTF-8'),
('nso_za', 'nso_ZA.UTF-8'),
('rw_rw', 'rw_RW.UTF-8'),
('ss_za', 'ss_ZA.UTF-8'),
('ta_in', 'ta_IN.UTF-8'),
('tn_za', 'tn_ZA.UTF-8'),
('ts_za', 'ts_ZA.UTF-8'),
('tt_ru', 'tt_RU.UTF-8'),
('ur_pk', 'ur_PK.UTF-8'),
('vi_vn', 'vi_VN.UTF-8'),
)


class TestHardcodedLocaleReplacements(unittest.TestCase):
"""UTF-8 preferences that override X11 legacy codesets.

See https://github.com/python/cpython/issues/151316
"""

@support.subTests('key,expected', _UTF8_X11_OVERRIDES)
def test_locale_alias_entries(self, key, expected):
self.assertEqual(locale.locale_alias[key], expected)

@support.subTests('key,expected', _UTF8_X11_OVERRIDES)
def test_normalize(self, key, expected):
self.assertEqual(locale.normalize(key), expected)
self.assertEqual(locale.normalize(key.upper()), expected)

@support.subTests('key,expected', _UTF8_X11_OVERRIDES)
def test_parse_localename(self, key, expected):
lang, encoding = expected.split('.')
self.assertEqual(locale._parse_localename(key), (lang, encoding))


class TestMiscellaneous(unittest.TestCase):
def test_defaults_UTF8(self):
# Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is
Expand Down
5 changes: 5 additions & 0 deletions Misc/NEWS.d/next/Library/2026-08-11-09-47-40.gh-issue-151316.M7ANwQ.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
:mod:`locale` now prefers UTF-8 for bare locale aliases that are
UTF-8-only in glibc (for example ``en_IN``, ``hi_IN``, ``es_CU``)
instead of obsolete X11 codesets. This restores
``setlocale(getlocale())`` round-trips on modern systems where those
locales have no legacy encoding.
23 changes: 22 additions & 1 deletion Tools/i18n/makelocalealias.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,25 @@ def parse_glibc_supported(filename):
data[locale] = alias
return data

def apply_x11_locales_patch(data, glibc_data):
"""Restore glibc UTF-8 defaults that X11 locale.alias overwrote.

Regeneration updates glibc SUPPORTED first, then X11 locale.alias.
X11 still maps several locales that are UTF-8-only in modern glibc to
obsolete encodings (for example en_IN -> en_IN.ISO8859-1). Prefer the
glibc UTF-8 mapping for those bare keys (gh-151316).

de_LI is special: glibc only ships de_LI.UTF-8 (no bare SUPPORTED line),
so the X11 de_LI.ISO8859-1 mapping would otherwise stick.
"""
for key, value in glibc_data.items():
if data.get(key) == value:
continue
if value.split('@')[0].endswith('.UTF-8'):
data[key] = value
data['de_li'] = 'de_LI.UTF-8'
return data

def pprint(data):
items = sorted(data.items())
for k, v in items:
Expand Down Expand Up @@ -149,11 +168,13 @@ def check(data):
args = parser.parse_args()

data = locale.locale_alias.copy()
data.update(parse_glibc_supported(args.glibc_supported))
glibc_data = parse_glibc_supported(args.glibc_supported)
data.update(glibc_data)
data.update(parse(args.locale_alias))
# Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist
# on all platforms.
data['c.utf8'] = 'C.UTF-8'
data = apply_x11_locales_patch(data, glibc_data)
while True:
# Repeat optimization while the size is decreased.
n = len(data)
Expand Down
Loading