From 48ef606b6641dc83fefa633c45c3be337d2f87e0 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 4 Jun 2026 23:02:07 +0200 Subject: [PATCH 1/3] cut: support multibyte characters in non-UTF-8 locales --- src/uu/cut/Cargo.toml | 2 +- src/uu/cut/locales/en-US.ftl | 1 + src/uu/cut/locales/fr-FR.ftl | 1 + src/uu/cut/src/cut.rs | 129 ++++++++++++- src/uucore/src/lib/features/i18n/charmap.rs | 29 ++- tests/by-util/test_cut.rs | 200 +++++++++++++++++++- 6 files changed, 347 insertions(+), 15 deletions(-) diff --git a/src/uu/cut/Cargo.toml b/src/uu/cut/Cargo.toml index b6bb74bfefc..a3106085690 100644 --- a/src/uu/cut/Cargo.toml +++ b/src/uu/cut/Cargo.toml @@ -17,7 +17,7 @@ doctest = false [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["ranges"] } +uucore = { workspace = true, features = ["ranges", "i18n-charmap"] } memchr = { workspace = true } bstr = { workspace = true } fluent = { workspace = true } diff --git a/src/uu/cut/locales/en-US.ftl b/src/uu/cut/locales/en-US.ftl index d320fc86d11..e2a9bce2a1a 100644 --- a/src/uu/cut/locales/en-US.ftl +++ b/src/uu/cut/locales/en-US.ftl @@ -101,6 +101,7 @@ cut-help-complement = invert the filter - instead of displaying only the filtere cut-help-only-delimited = in field mode, only print lines which contain the delimiter cut-help-zero-terminated = instead of filtering columns based on line, filter columns based on \\0 (NULL character) cut-help-output-delimiter = in field mode, replace the delimiter in output lines with this option's argument +cut-help-no-partial = with -b, don't output partial multi-byte characters # Error messages cut-error-is-directory = Is a directory diff --git a/src/uu/cut/locales/fr-FR.ftl b/src/uu/cut/locales/fr-FR.ftl index a95773099d6..be73c47b0ac 100644 --- a/src/uu/cut/locales/fr-FR.ftl +++ b/src/uu/cut/locales/fr-FR.ftl @@ -101,6 +101,7 @@ cut-help-complement = inverser le filtre - au lieu d'afficher seulement les colo cut-help-only-delimited = en mode champ, afficher seulement les lignes qui contiennent le délimiteur cut-help-zero-terminated = au lieu de filtrer les colonnes basées sur la ligne, filtrer les colonnes basées sur \\0 (caractère NULL) cut-help-output-delimiter = en mode champ, remplacer le délimiteur dans les lignes de sortie avec l'argument de cette option +cut-help-no-partial = avec -b, ne pas afficher les caractères multi-octets partiels # Messages d'erreur cut-error-is-directory = Est un répertoire diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 3457290f0eb..1fff84b8e04 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -13,6 +13,7 @@ use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, std use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError, set_exit_code}; +use uucore::i18n::charmap::{is_multibyte_locale, mb_char_len}; use uucore::line_ending::LineEnding; use uucore::os_str_as_bytes; @@ -29,6 +30,8 @@ struct Options<'a> { out_delimiter: Option<&'a [u8]>, line_ending: LineEnding, field_opts: Option>, + /// `-n`: with `-b`, do not split multi-byte characters across the selection. + suppress_split: bool, } enum Delimiter<'a> { @@ -104,6 +107,102 @@ fn cut_bytes( Ok(()) } +/// Walk `line` from byte offset `idx` (at position `pos`) as long as the +/// position of the last consumed character stays within `limit`, and return the +/// byte offset and position reached. +/// +/// The position is the number of characters seen so far when `by_char` is set +/// (`-c`), or the offset of the last byte consumed otherwise (`-b -n`). +fn advance( + line: &[u8], + mut idx: usize, + mut pos: usize, + limit: usize, + by_char: bool, +) -> (usize, usize) { + while pos < limit && idx < line.len() { + // ASCII bytes are single-byte characters in every encoding handled + // here, which keeps the common case out of the decoder. + let len = if line[idx] < 0x80 { + 1 + } else { + mb_char_len(&line[idx..]).max(1) // never exceeds the length left + }; + let step = if by_char { 1 } else { len }; + if pos + step > limit { + break; + } + idx += len; + pos += step; + } + (idx, pos) +} + +/// Cut `-c` (whole characters) or `-b -n` (bytes, keeping whole characters). +/// +/// In a single-byte locale, or for `-b` without `-n`, this falls back to the +/// plain byte path. Otherwise each character is emitted whole when its 1-based +/// position falls in a range: the character index for `-c`, or the offset of +/// its last byte for `-b -n` (matching GNU). +fn cut_chars( + reader: R, + out: &mut W, + ranges: &[Range], + opts: &Options, + by_char: bool, +) -> UResult<()> { + if !is_multibyte_locale() || !(by_char || opts.suppress_split) { + return cut_bytes(reader, out, ranges, opts); + } + + let newline_char = opts.line_ending.into(); + let mut buf_in = BufReader::new(reader); + let out_delim = opts.out_delimiter.unwrap_or(b"\t"); + + let result = buf_in.for_byte_record(newline_char, |line| { + let mut print_delim = false; + // Byte offset of the next character to look at, and the position + // already consumed. `ranges` is sorted and disjoint, so one pass over + // the line is enough and each range maps to a contiguous slice of it. + let (mut idx, mut pos) = (0, 0); + for &Range { low, high } in ranges { + // Skip the characters located before the range, then take the ones + // it covers: a character belongs to the range when its position is + // in `low..=high`. + (idx, pos) = advance(line, idx, pos, low - 1, by_char); + let start = idx; + if high >= line.len() { + // A character never ends before its own byte offset, so a range + // reaching past the end of the line covers everything left: + // there is no need to decode it. + idx = line.len(); + } else { + (idx, pos) = advance(line, idx, pos, high, by_char); + } + if start < idx { + if print_delim { + out.write_all(out_delim)?; + } else if opts.out_delimiter.is_some() { + print_delim = true; + } + out.write_all(&line[start..idx])?; + } + if idx == line.len() { + // The rest of the ranges start further away, past the line end. + break; + } + } + out.write_all(&[newline_char])?; + Ok(true) + }); + + if let Err(e) = result { + return Err(USimpleError::new(1, e.to_string())); + } + + Ok(()) +} + /// Output delimiter is explicitly specified fn cut_fields_explicit_out_delim( reader: R, @@ -458,8 +557,8 @@ where } show_if_err!(match mode { - Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => - cut_bytes(stdin(), &mut out, ranges, opts), + Mode::Bytes(ranges, opts) => cut_chars(stdin(), &mut out, ranges, opts, false), + Mode::Characters(ranges, opts) => cut_chars(stdin(), &mut out, ranges, opts, true), Mode::Fields(ranges, opts) => cut_fields(stdin(), &mut out, ranges, opts), }); @@ -482,8 +581,11 @@ where .map_err_context(|| filename.maybe_quote().to_string()) .and_then(|file| { match &mode { - Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => { - cut_bytes(file, &mut out, ranges, opts) + Mode::Bytes(ranges, opts) => { + cut_chars(file, &mut out, ranges, opts, false) + } + Mode::Characters(ranges, opts) => { + cut_chars(file, &mut out, ranges, opts, true) } Mode::Fields(ranges, opts) => cut_fields(file, &mut out, ranges, opts), } @@ -514,12 +616,14 @@ fn get_delimiters(matches: &ArgMatches) -> UResult<(Delimiter<'_>, Option<&[u8]> if os_string.is_empty() { Delimiter::Slice(b"\0") } else { - // For delimiter `-d` option value - allow both UTF-8 (possibly multi-byte) characters - // and Non UTF-8 (and not ASCII) single byte "characters", like `b"\xAD"` to align with GNU behavior + // The delimiter must be a single character. We accept a single + // UTF-8 character (e.g. an emoji), a single byte (including a + // non-UTF-8 byte like `b"\xFF"`), or a single character of the + // current locale's encoding (e.g. a 2-byte GB18030 character). let bytes = os_str_as_bytes(os_string)?; - if os_string.to_str().is_some_and(|s| s.chars().count() > 1) - || os_string.to_str().is_none() && bytes.len() > 1 - { + let single_utf8_char = os_string.to_str().is_some_and(|s| s.chars().count() == 1); + let single_locale_char = mb_char_len(bytes) == bytes.len(); + if !single_utf8_char && !single_locale_char { return Err(USimpleError::new( 1, translate!("cut-error-delimiter-must-be-single-character"), @@ -583,6 +687,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let (delimiter, out_delimiter) = get_delimiters(&matches)?; let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)); + let suppress_split = matches.get_flag(options::NOTHING); // Only one, and only one of cutting mode arguments, i.e. `-b`, `-c`, `-f`, // is expected. The number of those arguments is used for parsing a cutting @@ -610,6 +715,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { out_delimiter, line_ending, field_opts: None, + suppress_split, }, ) }) @@ -623,6 +729,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { out_delimiter, line_ending, field_opts: None, + suppress_split, }, ) }) @@ -639,6 +746,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { delimiter, only_delimited, }), + suppress_split, }, ) }) @@ -776,7 +884,8 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::NOTHING) .short('n') - .help("(ignored)") + .long("no-partial") + .help(translate!("cut-help-no-partial")) .action(ArgAction::SetTrue), ) } diff --git a/src/uucore/src/lib/features/i18n/charmap.rs b/src/uucore/src/lib/features/i18n/charmap.rs index 2ec99229bc8..f1d60e3055b 100644 --- a/src/uucore/src/lib/features/i18n/charmap.rs +++ b/src/uucore/src/lib/features/i18n/charmap.rs @@ -10,6 +10,7 @@ use std::sync::OnceLock; enum MbEncoding { + SingleByte, Utf8, Gb18030, EucJp, @@ -19,11 +20,12 @@ enum MbEncoding { fn encoding_from_name(enc: &str) -> MbEncoding { match enc { + "utf-8" | "utf8" => MbEncoding::Utf8, "gb18030" | "gbk" | "gb2312" => MbEncoding::Gb18030, "euc-jp" | "eucjp" => MbEncoding::EucJp, "euc-kr" | "euckr" => MbEncoding::EucKr, "big5" | "big5-hkscs" | "big5hkscs" | "euc-tw" | "euctw" => MbEncoding::Big5, - _ => MbEncoding::Utf8, + _ => MbEncoding::SingleByte, } } @@ -35,7 +37,7 @@ fn get_encoding() -> &'static MbEncoding { .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty())); let s = match val.as_deref() { Some(s) if s != "C" && s != "POSIX" => s, - _ => return MbEncoding::Utf8, + _ => return MbEncoding::SingleByte, }; if let Some(enc) = s.split('.').nth(1) { let enc = enc.split('@').next().unwrap_or(enc); @@ -51,6 +53,12 @@ fn get_encoding() -> &'static MbEncoding { }) } +/// Whether the current locale uses a multi-byte encoding (i.e. `MB_CUR_MAX > 1`). +/// `C`/`POSIX` and single-byte encodings return `false`. +pub fn is_multibyte_locale() -> bool { + !matches!(get_encoding(), MbEncoding::SingleByte) +} + /// Byte length of the first character in `bytes` under the current locale encoding. pub fn mb_char_len(bytes: &[u8]) -> usize { debug_assert!(!bytes.is_empty()); @@ -59,7 +67,9 @@ pub fn mb_char_len(bytes: &[u8]) -> usize { return 1; } match get_encoding() { - MbEncoding::Utf8 => utf8_len(bytes, b0), + // `C`/`POSIX` and unknown encodings have `MB_CUR_MAX == 1`, but we still + // decode UTF-8 there as a sensible default for byte-length detection. + MbEncoding::SingleByte | MbEncoding::Utf8 => utf8_len(bytes, b0), MbEncoding::Gb18030 => gb18030_len(bytes, b0), MbEncoding::EucJp => eucjp_len(bytes, b0), MbEncoding::EucKr => euckr_len(bytes, b0), @@ -67,6 +77,19 @@ pub fn mb_char_len(bytes: &[u8]) -> usize { } } +/// Iterate over the characters of `bytes` under the current locale encoding, +/// yielding each character as a byte slice. Invalid bytes are yielded one at a +/// time, so the concatenation of all items is always `bytes`. +pub fn mb_chars(bytes: &[u8]) -> impl Iterator { + let mut idx = 0; + std::iter::from_fn(move || { + let rest = bytes.get(idx..).filter(|r| !r.is_empty())?; + let len = mb_char_len(rest).max(1); // mb_char_len never exceeds rest.len() + idx += len; + Some(&bytes[idx - len..idx]) + }) +} + // All helpers below assume b0 > 0x7F (ASCII already handled by caller). fn utf8_len(b: &[u8], b0: u8) -> usize { diff --git a/tests/by-util/test_cut.rs b/tests/by-util/test_cut.rs index 563c752b115..9f89a34e8bf 100644 --- a/tests/by-util/test_cut.rs +++ b/tests/by-util/test_cut.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore defg +// spell-checker:ignore defg naïve nave use uutests::{at_and_ucmd, new_ucmd}; @@ -660,3 +660,201 @@ fn test_cut_non_utf8_paths() { .succeeds() .stdout_only("a\tc\n1\t3\n"); } + +// We exercise the GB18030 path with two real two-byte characters that are not +// valid UTF-8: 啊 (0xB0 0xA1) and 中 (0xD6 0xD0). The active encoding comes +// straight from `LC_ALL`, so the host does not need the locale installed. +#[cfg(target_os = "linux")] +const GB_LOCALE: &str = "zh_CN.gb18030"; +#[cfg(target_os = "linux")] +const A: &[u8] = b"\xB0\xA1"; // 啊 + +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_fields_gb18030_delimiter() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + // Three words separated by the two-byte 啊: "red啊green啊blue". + let line = b"red\xB0\xA1green\xB0\xA1blue\n"; + let delim = OsString::from_vec(A.to_vec()); + + // Pick the last field; the chosen output delimiter replaces the input one. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-d") + .arg(&delim) + .args(&["-f3", "--output-delimiter=/"]) + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only("blue\n"); + + // Two non-adjacent fields; with no override the multibyte delimiter itself + // is re-emitted between them. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-d") + .arg(&delim) + .arg("-f1,3") + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only_bytes(b"red\xB0\xA1blue\n"); +} + +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_fields_gb18030_complement_and_gaps() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let delim = OsString::from_vec(A.to_vec()); + + // --complement of the middle field leaves the two outer ones, rejoined. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("--complement") + .arg("-d") + .arg(&delim) + .arg("-f2") + .pipe_in(b"red\xB0\xA1green\xB0\xA1blue\n".to_vec()) + .succeeds() + .stdout_only_bytes(b"red\xB0\xA1blue\n"); + + // A line that is only delimiters yields empty fields around a trailing one. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-d") + .arg(&delim) + .args(&["-f1-3", "--output-delimiter=|"]) + .pipe_in(b"\xB0\xA1\xB0\xA1z\n".to_vec()) + .succeeds() + .stdout_only("||z\n"); +} + +#[test] +#[cfg(target_os = "linux")] +fn test_cut_fields_single_byte_delimiter_in_mb_locale() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + // 0x80 never starts a GB18030 sequence, yet a lone byte is a fine delimiter. + let delim = OsString::from_vec(vec![0x80]); + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-d") + .arg(&delim) + .args(&["-f1,3", "--output-delimiter=-"]) + .pipe_in(b"a\x80b\x80c\n".to_vec()) + .succeeds() + .stdout_only("a-c\n"); +} + +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_chars_gb18030() { + // "啊w中": -c counts characters, so the second one is the ASCII 'w'. + let line = b"\xB0\xA1w\xD6\xD0\n"; + + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-c2") + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only("w\n"); + + // Selecting the trailing multibyte character returns it whole. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-c3") + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only_bytes(b"\xD6\xD0\n"); + + // A range that spans the leading and ASCII characters. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .arg("-c1-2") + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only_bytes(b"\xB0\xA1w\n"); +} + +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_chars_gb18030_ranges_and_complement() { + // "啊w中": list of two single-char ranges joined by a custom delimiter. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .args(&["-c1,3", "--output-delimiter=+"]) + .pipe_in(b"\xB0\xA1w\xD6\xD0\n".to_vec()) + .succeeds() + .stdout_only_bytes(b"\xB0\xA1+\xD6\xD0\n"); + + // Complement of the ASCII middle character keeps both multibyte ones. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .args(&["--complement", "-c2"]) + .pipe_in(b"\xB0\xA1w\xD6\xD0\n".to_vec()) + .succeeds() + .stdout_only_bytes(b"\xB0\xA1\xD6\xD0\n"); +} + +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_bytes_no_split_gb18030() { + // -n forbids splitting a multibyte character: a byte index landing inside + // 啊 only produces output once its final byte is included. + let line = b"\xB0\xA1w\n"; + + // Byte 1 is the first half of 啊 -> nothing is emitted. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .args(&["-b1", "-n"]) + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only("\n"); + + // Byte 2 completes 啊 -> the whole character comes out. + new_ucmd!() + .env("LC_ALL", GB_LOCALE) + .args(&["-b2", "-n"]) + .pipe_in(line.to_vec()) + .succeeds() + .stdout_only_bytes(b"\xB0\xA1\n"); +} + +// `-c` also operates on whole characters in a UTF-8 locale. The harness runs +// under `LC_ALL=C`, so the locale is forced here. "naïve" is n a ï(2 bytes) v e. +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_chars_utf8() { + // The third character is the accented 'ï', returned in full. + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .arg("-c3") + .pipe_in("naïve\n".as_bytes().to_vec()) + .succeeds() + .stdout_only("ï\n"); + + // Complement of that character removes it and nothing else. + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["--complement", "-c3"]) + .pipe_in("naïve\n".as_bytes().to_vec()) + .succeeds() + .stdout_only("nave\n"); + + // A list straddling 'ï' joins the two picked characters. + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["-c1,4", "--output-delimiter=+"]) + .pipe_in("naïve\n".as_bytes().to_vec()) + .succeeds() + .stdout_only("n+v\n"); +} From a55030d91d310e1d9de07b44fea6c2ebc8d1d2ca Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 4 Jul 2026 23:45:51 +0200 Subject: [PATCH 2/3] cut: skip WASI-incompatible non-UTF-8 delimiter test under wasi_runner wasmtime's argument marshaling requires valid UTF-8, so a raw non-UTF-8 byte delimiter can't be passed through to the WASI binary the way it can natively on Linux. --- tests/by-util/test_cut.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/by-util/test_cut.rs b/tests/by-util/test_cut.rs index 9f89a34e8bf..af9382a4378 100644 --- a/tests/by-util/test_cut.rs +++ b/tests/by-util/test_cut.rs @@ -735,6 +735,10 @@ fn test_cut_fields_gb18030_complement_and_gaps() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: non-UTF-8 arguments can't be passed through wasmtime" +)] fn test_cut_fields_single_byte_delimiter_in_mb_locale() { use std::ffi::OsString; use std::os::unix::ffi::OsStringExt; From 64aa9be60b1f4f799810b4e3abc78ef8bb0705fc Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 30 Jul 2026 22:08:00 +0200 Subject: [PATCH 3/3] cut: speed up the multibyte character path The character path walked the line one character at a time through the encoding decoder and looked the locale encoding up per character, which dominated the per-line cost on mixed ASCII and multi-byte text. Take runs of ASCII bytes a machine word at a time. They are single-byte characters in every encoding handled here, so byte offset and character position move together across a run, and only the bytes above 0x7F need the decoder. Bundle the parts that are fixed for the whole run -- the ranges, the output delimiter and whether it was given, the position mode and the encoding -- into a CharCut built once by cut_chars, and make the line body a method on it, so the per-line call passes one pointer rather than seven arguments. In uucore, MbEncoding becomes the public Encoding and locale_encoding() returns it by value, so a caller decoding many characters resolves the locale once and keeps it in a register instead of reaching through a OnceLock per character. is_multibyte_locale() had no callers left. Instruction counts against the parent commit (cachegrind, LC_ALL=C.UTF-8): -c 5-30, 100k mixed short lines 30.44M -> 28.95M -c 20-70, 20k long multibyte lines 44.38M -> 41.02M That is roughly 1.1x in wall clock on both shapes; the machine was too loaded to quote a tighter figure. The single-byte path (LC_ALL=C) and field mode are unchanged. Selecting a range of characters still costs more than the same range of bytes, and always will: -c used to be an alias for -b, and characters have to be decoded to be counted. Tests cover advance directly -- character counting for -c, byte counting for -b -n, and the word boundary crossings -- plus a cut -c case over mixed ASCII and multi-byte lines. --- src/uu/cut/src/cut.rs | 289 +++++++++++++++----- src/uucore/src/lib/features/i18n/charmap.rs | 106 +++---- tests/by-util/test_cut.rs | 25 ++ 3 files changed, 297 insertions(+), 123 deletions(-) diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 1fff84b8e04..a328c3bcabd 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -13,7 +13,7 @@ use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, std use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError, set_exit_code}; -use uucore::i18n::charmap::{is_multibyte_locale, mb_char_len}; +use uucore::i18n::charmap::{Encoding, locale_encoding, mb_char_len}; use uucore::line_ending::LineEnding; use uucore::os_str_as_bytes; @@ -70,6 +70,37 @@ fn list_to_ranges(list: &str, complement: bool) -> Result, String> { } } +/// Write the parts of `line` selected by `ranges`, treating every byte as a +/// character. +/// +/// Always inlined: it is the body of the per-line loop, and a call per line +/// costs more than the work it does on short lines. +#[inline(always)] +fn write_line_bytes( + line: &[u8], + out: &mut W, + ranges: &[Range], + out_delim: &[u8], + explicit_delim: bool, +) -> std::io::Result<()> { + let mut print_delim = false; + for &Range { low, high } in ranges { + if low > line.len() { + break; + } + if print_delim { + out.write_all(out_delim)?; + } else if explicit_delim { + print_delim = true; + } + // change `low` from 1-indexed value to 0-index value + let low = low - 1; + let high = high.min(line.len()); + out.write_all(&line[low..high])?; + } + Ok(()) +} + fn cut_bytes( reader: R, out: &mut W, @@ -79,23 +110,10 @@ fn cut_bytes( let newline_char = opts.line_ending.into(); let mut buf_in = BufReader::new(reader); let out_delim = opts.out_delimiter.unwrap_or(b"\t"); + let explicit_delim = opts.out_delimiter.is_some(); let result = buf_in.for_byte_record(newline_char, |line| { - let mut print_delim = false; - for &Range { low, high } in ranges { - if low > line.len() { - break; - } - if print_delim { - out.write_all(out_delim)?; - } else if opts.out_delimiter.is_some() { - print_delim = true; - } - // change `low` from 1-indexed value to 0-index value - let low = low - 1; - let high = high.min(line.len()); - out.write_all(&line[low..high])?; - } + write_line_bytes(line, out, ranges, out_delim, explicit_delim)?; out.write_all(&[newline_char])?; Ok(true) }); @@ -107,35 +125,117 @@ fn cut_bytes( Ok(()) } -/// Walk `line` from byte offset `idx` (at position `pos`) as long as the -/// position of the last consumed character stays within `limit`, and return the -/// byte offset and position reached. +/// Offset of the first byte above `0x7F` in `bytes`, or `bytes.len()` if there +/// is none. /// -/// The position is the number of characters seen so far when `by_char` is set -/// (`-c`), or the offset of the last byte consumed otherwise (`-b -n`). -fn advance( - line: &[u8], - mut idx: usize, - mut pos: usize, - limit: usize, +/// Whole words are scanned at a time while at least one is left, then the tail +/// byte by byte. Callers pass exactly the bytes they may consume, so the word +/// loop never has to discount bytes reaching past the end. +#[inline(always)] +fn ascii_run(bytes: &[u8]) -> usize { + const HIGH_BITS: u64 = 0x8080_8080_8080_8080; + + let mut idx = 0; + while let Some(chunk) = bytes[idx..].first_chunk::<8>() { + let high = u64::from_le_bytes(*chunk) & HIGH_BITS; + if high != 0 { + return idx + high.trailing_zeros() as usize / 8; + } + idx += 8; + } + while idx < bytes.len() && bytes[idx] < 0x80 { + idx += 1; + } + idx +} + +/// Everything the character-mode line loop needs besides the line itself. It is +/// all fixed for the whole run, so it is built once and passed by reference: +/// handing these over one argument per line costs more than the work done on a +/// short line. +struct CharCut<'a> { + ranges: &'a [Range], + out_delim: &'a [u8], + explicit_delim: bool, + /// `-c`: a position is the index of the character. Otherwise (`-b -n`) it + /// is the offset of the character's last byte, which is what GNU selects on. by_char: bool, -) -> (usize, usize) { - while pos < limit && idx < line.len() { - // ASCII bytes are single-byte characters in every encoding handled - // here, which keeps the common case out of the decoder. - let len = if line[idx] < 0x80 { - 1 - } else { - mb_char_len(&line[idx..]).max(1) // never exceeds the length left - }; - let step = if by_char { 1 } else { len }; - if pos + step > limit { - break; + encoding: Encoding, +} + +impl CharCut<'_> { + /// Walk `line` from byte offset `idx` (at position `pos`) as long as the + /// position of the last consumed character stays within `limit`, and return + /// the byte offset and position reached. + #[inline(always)] + fn advance(&self, line: &[u8], mut idx: usize, mut pos: usize, limit: usize) -> (usize, usize) { + while pos < limit && idx < line.len() { + // ASCII bytes are single-byte characters in every encoding handled + // here, so offset and position move together and a whole run of + // them can be taken at once, without going through the decoder. + let room = (limit - pos).min(line.len() - idx); + let run = ascii_run(&line[idx..idx + room]); + idx += run; + pos += run; + if run == room { + break; + } + let len = self.encoding.char_len(&line[idx..]); // in `1..=line.len() - idx` + let step = if self.by_char { 1 } else { len }; + if pos + step > limit { + break; + } + idx += len; + pos += step; } - idx += len; - pos += step; + (idx, pos) + } + + /// Write the parts of `line` selected by the ranges, keeping multi-byte + /// characters whole. A character belongs to a range when its position is in + /// `low..=high`. + /// + /// Always inlined, like its byte counterpart: it is the body of the + /// per-line loop, and the call costs more than the work done on a short + /// line. + #[inline(always)] + fn write_line(&self, line: &[u8], out: &mut W) -> std::io::Result<()> { + let mut print_delim = false; + // Byte offset of the next character to look at, and the position + // already consumed. The ranges are sorted and disjoint, so one pass + // over the line is enough and each range maps to a contiguous slice. + let (mut idx, mut pos) = (0, 0); + for &Range { low, high } in self.ranges { + // A character position is never below its own byte offset, so a + // range starting past the last byte selects nothing, and so do the + // ones after it. + if low > line.len() { + break; + } + // Skip the characters located before the range. + (idx, pos) = self.advance(line, idx, pos, low - 1); + if idx == line.len() { + break; + } + if print_delim { + out.write_all(self.out_delim)?; + } else if self.explicit_delim { + print_delim = true; + } + let start = idx; + if high >= line.len() { + // The range reaches past the end of the line, so it covers + // every character left and none of them needs to be decoded. + // The ranges after it start further away still. + return out.write_all(&line[start..]); + } + // At least one character is taken: `pos` is below `high` and there + // are bytes left, so this always moves `idx` forward. + (idx, pos) = self.advance(line, idx, pos, high); + out.write_all(&line[start..idx])?; + } + Ok(()) } - (idx, pos) } /// Cut `-c` (whole characters) or `-b -n` (bytes, keeping whole characters). @@ -151,47 +251,23 @@ fn cut_chars( opts: &Options, by_char: bool, ) -> UResult<()> { - if !is_multibyte_locale() || !(by_char || opts.suppress_split) { + let encoding = locale_encoding(); + if encoding == Encoding::SingleByte || !(by_char || opts.suppress_split) { return cut_bytes(reader, out, ranges, opts); } let newline_char = opts.line_ending.into(); let mut buf_in = BufReader::new(reader); - let out_delim = opts.out_delimiter.unwrap_or(b"\t"); + let cut = CharCut { + ranges, + out_delim: opts.out_delimiter.unwrap_or(b"\t"), + explicit_delim: opts.out_delimiter.is_some(), + by_char, + encoding, + }; let result = buf_in.for_byte_record(newline_char, |line| { - let mut print_delim = false; - // Byte offset of the next character to look at, and the position - // already consumed. `ranges` is sorted and disjoint, so one pass over - // the line is enough and each range maps to a contiguous slice of it. - let (mut idx, mut pos) = (0, 0); - for &Range { low, high } in ranges { - // Skip the characters located before the range, then take the ones - // it covers: a character belongs to the range when its position is - // in `low..=high`. - (idx, pos) = advance(line, idx, pos, low - 1, by_char); - let start = idx; - if high >= line.len() { - // A character never ends before its own byte offset, so a range - // reaching past the end of the line covers everything left: - // there is no need to decode it. - idx = line.len(); - } else { - (idx, pos) = advance(line, idx, pos, high, by_char); - } - if start < idx { - if print_delim { - out.write_all(out_delim)?; - } else if opts.out_delimiter.is_some() { - print_delim = true; - } - out.write_all(&line[start..idx])?; - } - if idx == line.len() { - // The rest of the ranges start further away, past the line end. - break; - } - } + cut.write_line(line, out)?; out.write_all(&[newline_char])?; Ok(true) }); @@ -889,3 +965,66 @@ pub fn uu_app() -> Command { .action(ArgAction::SetTrue), ) } + +#[cfg(test)] +mod tests { + use super::{CharCut, Encoding}; + + fn utf8_cut(by_char: bool) -> CharCut<'static> { + CharCut { + ranges: &[], + out_delim: b"\t", + explicit_delim: false, + by_char, + encoding: Encoding::Utf8, + } + } + + // "quick" + 2-byte char + "brown" + 3-byte char + "foxjumping" + const LINE: &[u8] = b"quick\xc3\xa9brown\xe2\x82\xacfoxjumping"; + + #[test] + fn advance_counts_characters_for_c() { + let cut = utf8_cut(true); + // Inside the leading ASCII run, and stopping right on its last byte. + assert_eq!(cut.advance(LINE, 0, 0, 3), (3, 3)); + assert_eq!(cut.advance(LINE, 0, 0, 5), (5, 5)); + // Taking the 2-byte character moves two bytes but one position. + assert_eq!(cut.advance(LINE, 0, 0, 6), (7, 6)); + // Through "brown": 12 bytes for 11 characters, the 2-byte one included. + assert_eq!(cut.advance(LINE, 0, 0, 11), (12, 11)); + // A limit past the end stops at the end: 25 bytes, 22 characters. + assert_eq!(cut.advance(LINE, 0, 0, 99), (LINE.len(), 22)); + // Resuming mid-line, and a limit that is already reached. + assert_eq!(cut.advance(LINE, 7, 6, 9), (10, 9)); + assert_eq!(cut.advance(LINE, 7, 6, 6), (7, 6)); + } + + #[test] + fn advance_counts_bytes_for_b_n() { + let cut = utf8_cut(false); + // Positions are byte offsets here, so the ASCII head is unchanged. + assert_eq!(cut.advance(LINE, 0, 0, 5), (5, 5)); + // The 2-byte character only fits once the limit covers both bytes. + assert_eq!(cut.advance(LINE, 0, 0, 6), (5, 5)); + assert_eq!(cut.advance(LINE, 0, 0, 7), (7, 7)); + // Likewise the 3-byte one: 12 and 13 are short, 14 takes it. + assert_eq!(cut.advance(LINE, 0, 0, 13), (12, 12)); + assert_eq!(cut.advance(LINE, 0, 0, 15), (15, 15)); + } + + #[test] + fn advance_crosses_word_boundaries() { + let cut = utf8_cut(true); + // "foxjumping" is long enough for the word-at-a-time path, both + // landing exactly on a word boundary and past one. + let tail = &LINE[15..]; + assert_eq!(cut.advance(tail, 0, 0, 8), (8, 8)); + assert_eq!(cut.advance(tail, 0, 0, 10), (10, 10)); + // A high byte found by the word scan still yields to the decoder. + let wide = b"abcdefghij\xc3\xa9kl"; + assert_eq!(cut.advance(wide, 0, 0, 11), (12, 11)); + // An empty line has nothing to walk. + assert_eq!(cut.advance(b"", 0, 0, 4), (0, 0)); + } +} diff --git a/src/uucore/src/lib/features/i18n/charmap.rs b/src/uucore/src/lib/features/i18n/charmap.rs index f1d60e3055b..2c8bc0d1006 100644 --- a/src/uucore/src/lib/features/i18n/charmap.rs +++ b/src/uucore/src/lib/features/i18n/charmap.rs @@ -9,7 +9,10 @@ use std::sync::OnceLock; -enum MbEncoding { +/// Character encoding of the current locale, as far as character *lengths* are +/// concerned. `SingleByte` covers `C`/`POSIX` and every 8-bit encoding. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Encoding { SingleByte, Utf8, Gb18030, @@ -18,26 +21,30 @@ enum MbEncoding { Big5, } -fn encoding_from_name(enc: &str) -> MbEncoding { +fn encoding_from_name(enc: &str) -> Encoding { match enc { - "utf-8" | "utf8" => MbEncoding::Utf8, - "gb18030" | "gbk" | "gb2312" => MbEncoding::Gb18030, - "euc-jp" | "eucjp" => MbEncoding::EucJp, - "euc-kr" | "euckr" => MbEncoding::EucKr, - "big5" | "big5-hkscs" | "big5hkscs" | "euc-tw" | "euctw" => MbEncoding::Big5, - _ => MbEncoding::SingleByte, + "utf-8" | "utf8" => Encoding::Utf8, + "gb18030" | "gbk" | "gb2312" => Encoding::Gb18030, + "euc-jp" | "eucjp" => Encoding::EucJp, + "euc-kr" | "euckr" => Encoding::EucKr, + "big5" | "big5-hkscs" | "big5hkscs" | "euc-tw" | "euctw" => Encoding::Big5, + _ => Encoding::SingleByte, } } -fn get_encoding() -> &'static MbEncoding { - static ENCODING: OnceLock = OnceLock::new(); - ENCODING.get_or_init(|| { +/// Encoding of the current locale, resolved from the environment on first use. +/// +/// Callers that decode more than one character should hold on to the returned +/// value: it turns the per-character encoding lookup into a register read. +pub fn locale_encoding() -> Encoding { + static ENCODING: OnceLock = OnceLock::new(); + *ENCODING.get_or_init(|| { let val = ["LC_ALL", "LC_CTYPE", "LANG"] .iter() .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty())); let s = match val.as_deref() { Some(s) if s != "C" && s != "POSIX" => s, - _ => return MbEncoding::SingleByte, + _ => return Encoding::SingleByte, }; if let Some(enc) = s.split('.').nth(1) { let enc = enc.split('@').next().unwrap_or(enc); @@ -45,67 +52,70 @@ fn get_encoding() -> &'static MbEncoding { } else { // Bare locale defaults from glibc localedata/SUPPORTED match s.split('@').next().unwrap_or(s) { - "zh_CN" | "zh_SG" => MbEncoding::Gb18030, - "zh_TW" | "zh_HK" => MbEncoding::Big5, - _ => MbEncoding::Utf8, + "zh_CN" | "zh_SG" => Encoding::Gb18030, + "zh_TW" | "zh_HK" => Encoding::Big5, + _ => Encoding::Utf8, } } }) } -/// Whether the current locale uses a multi-byte encoding (i.e. `MB_CUR_MAX > 1`). -/// `C`/`POSIX` and single-byte encodings return `false`. -pub fn is_multibyte_locale() -> bool { - !matches!(get_encoding(), MbEncoding::SingleByte) +impl Encoding { + /// Byte length of the first character in `bytes`. Never returns more than + /// `bytes.len()`, and never `0` for a non-empty slice. + #[inline] + pub fn char_len(self, bytes: &[u8]) -> usize { + debug_assert!(!bytes.is_empty()); + let b0 = bytes[0]; + if b0 <= 0x7F { + return 1; + } + match self { + // `C`/`POSIX` and unknown encodings have `MB_CUR_MAX == 1`, but we + // still decode UTF-8 there as a sensible default for byte-length + // detection. + Self::SingleByte | Self::Utf8 => utf8_len(bytes, b0), + Self::Gb18030 => gb18030_len(bytes, b0), + Self::EucJp => eucjp_len(bytes, b0), + Self::EucKr => euckr_len(bytes, b0), + Self::Big5 => big5_len(bytes, b0), + } + } } /// Byte length of the first character in `bytes` under the current locale encoding. pub fn mb_char_len(bytes: &[u8]) -> usize { - debug_assert!(!bytes.is_empty()); - let b0 = bytes[0]; - if b0 <= 0x7F { - return 1; - } - match get_encoding() { - // `C`/`POSIX` and unknown encodings have `MB_CUR_MAX == 1`, but we still - // decode UTF-8 there as a sensible default for byte-length detection. - MbEncoding::SingleByte | MbEncoding::Utf8 => utf8_len(bytes, b0), - MbEncoding::Gb18030 => gb18030_len(bytes, b0), - MbEncoding::EucJp => eucjp_len(bytes, b0), - MbEncoding::EucKr => euckr_len(bytes, b0), - MbEncoding::Big5 => big5_len(bytes, b0), - } -} - -/// Iterate over the characters of `bytes` under the current locale encoding, -/// yielding each character as a byte slice. Invalid bytes are yielded one at a -/// time, so the concatenation of all items is always `bytes`. -pub fn mb_chars(bytes: &[u8]) -> impl Iterator { - let mut idx = 0; - std::iter::from_fn(move || { - let rest = bytes.get(idx..).filter(|r| !r.is_empty())?; - let len = mb_char_len(rest).max(1); // mb_char_len never exceeds rest.len() - idx += len; - Some(&bytes[idx - len..idx]) - }) + locale_encoding().char_len(bytes) } // All helpers below assume b0 > 0x7F (ASCII already handled by caller). fn utf8_len(b: &[u8], b0: u8) -> usize { + // Two-byte sequences are by far the most common outside ASCII, so they get + // a single continuation-byte test rather than a loop. + if matches!(b0, 0xC2..=0xDF) { + return if b.len() >= 2 && is_continuation(b[1]) { + 2 + } else { + 1 + }; + } let n = match b0 { - 0xC2..=0xDF => 2, 0xE0..=0xEF => 3, 0xF0..=0xF4 => 4, _ => return 1, }; - if b.len() >= n && b[1..n].iter().all(|&c| c & 0xC0 == 0x80) { + if b.len() >= n && b[1..n].iter().copied().all(is_continuation) { n } else { 1 } } +fn is_continuation(byte: u8) -> bool { + byte & 0xC0 == 0x80 +} + // 2-byte: [81-FE][40-7E,80-FE] 4-byte: [81-FE][30-39][81-FE][30-39] fn gb18030_len(b: &[u8], b0: u8) -> usize { if !(0x81..=0xFE).contains(&b0) { diff --git a/tests/by-util/test_cut.rs b/tests/by-util/test_cut.rs index af9382a4378..418b692497d 100644 --- a/tests/by-util/test_cut.rs +++ b/tests/by-util/test_cut.rs @@ -862,3 +862,28 @@ fn test_cut_chars_utf8() { .succeeds() .stdout_only("n+v\n"); } + +// Lines with no byte above 0x7F take a byte-wise shortcut in a multi-byte +// locale; mixing them with multi-byte ones checks both paths agree on offsets. +#[test] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv must be valid UTF-8")] +fn test_cut_chars_utf8_mixed_ascii_lines() { + let input = "quokka\nfjärd\nwombat\ntøys\n"; + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .arg("-c3-5") + .pipe_in(input) + .succeeds() + .stdout_only("okk\närd\nmba\nys\n"); + + // `-b -n` selects a character when its last byte falls in the range, so the + // accented lines cover a different span than the ASCII ones. + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["-b", "3-5", "-n"]) + .pipe_in(input) + .succeeds() + .stdout_only("okk\när\nmba\nøys\n"); +}