From 1e42316c8fbbaf91c26557a8126b815170fc4298 Mon Sep 17 00:00:00 2001 From: liuweiqing Date: Mon, 6 Jul 2026 23:20:15 +0800 Subject: [PATCH] fix(tokenizer): treat special-token strings as text to prevent batch crashes ASR models such as Fun-ASR-Nano occasionally emit special-token strings (e.g. <|no|>, a language tag) as part of their transcription text. When that text is re-encoded via Tokenizer.encode for downstream tasks (forced alignment, loss computation), tiktoken rejects it by default (disallowed_special="all"), crashing the whole batch on a single bad sample (issue #3110). Tokenizer.encode now defaults disallowed_special=() so special-token strings are treated as ordinary text. Callers that explicitly pass allowed_special / disallowed_special are left untouched (setdefault), so decoding.py's encode(prompt, allowed_special="all") still encodes special tokens as their token ids, and any caller wanting the strict behaviour can still opt in. Adds tests/test_sensevoice_tokenizer_special_tokens.py with a minimal in-memory tiktoken encoding (no model download or GPU required). Fixes #3110 --- .../sense_voice/whisper_lib/tokenizer.py | 17 ++- ...est_sensevoice_tokenizer_special_tokens.py | 110 ++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/test_sensevoice_tokenizer_special_tokens.py diff --git a/funasr/models/sense_voice/whisper_lib/tokenizer.py b/funasr/models/sense_voice/whisper_lib/tokenizer.py index 4a7520519..4722b8f18 100644 --- a/funasr/models/sense_voice/whisper_lib/tokenizer.py +++ b/funasr/models/sense_voice/whisper_lib/tokenizer.py @@ -194,11 +194,24 @@ def __post_init__(self): def encode(self, text, **kwargs): """Encode. - + Args: text: Text tensor or string input. **kwargs: Additional keyword arguments. - """ + + ``tiktoken`` rejects any text containing a special-token string + (e.g. ``<|no|>``, ``<|zh|>``, ``<|nospeech|>``) by default + (``disallowed_special="all"``). ASR models such as Fun-ASR-Nano + occasionally emit such special-token strings as part of their + transcription text; re-encoding that text for downstream tasks + (forced alignment, loss computation, ...) then crashes the whole + batch with a single bad sample (see issue #3110). + + Treat special-token strings as ordinary text unless the caller + explicitly opts into a stricter policy. Callers that already pass + ``allowed_special`` / ``disallowed_special`` are left untouched. + """ + kwargs.setdefault("disallowed_special", ()) return self.encoding.encode(text, **kwargs) def decode(self, token_ids: List[int], **kwargs) -> str: diff --git a/tests/test_sensevoice_tokenizer_special_tokens.py b/tests/test_sensevoice_tokenizer_special_tokens.py new file mode 100644 index 000000000..73c3ab4fe --- /dev/null +++ b/tests/test_sensevoice_tokenizer_special_tokens.py @@ -0,0 +1,110 @@ +"""Unit tests for SenseVoice Tokenizer handling of special-token strings. + +Regression guard for issue #3110: ASR models such as Fun-ASR-Nano occasionally +emit special-token strings (e.g. ``<|no|>``, a language tag) as part of the +transcription text. Re-encoding that text via ``tiktoken.Encoding.encode`` +crashes by default (``disallowed_special="all"``), which takes down the whole +batch on a single bad sample during forced alignment / loss computation. +``Tokenizer.encode`` now treats special-token strings as ordinary text unless +the caller explicitly opts into a stricter policy. + +These tests build a minimal in-memory tiktoken encoding, so they run without +the Fun-ASR-Nano ``multilingual.tiktoken`` vocab (which ships with the model, +not this repo) and without a GPU or model download. +""" + +import unittest + +import tiktoken + +from funasr.models.sense_voice.whisper_lib.tokenizer import Tokenizer + + +def _build_test_encoding() -> tiktoken.Encoding: + """A minimal byte-level BPE encoding with the specials Tokenizer needs. + + Every single byte is its own token, so arbitrary text can be encoded + without the model's multilingual.tiktoken vocab file. + """ + mergeable_ranks = {bytes([b]): b for b in range(256)} + n_vocab = len(mergeable_ranks) + + # Specials required by Tokenizer.__post_init__ (startoftranscript/translate/ + # transcribe) plus a few language tags. "<|no|>" is the Norwegian tag that + # triggered #3110. + required_specials = [ + "<|startoftranscript|>", + "<|no|>", + "<|zh|>", + "<|en|>", + "<|translate|>", + "<|transcribe|>", + "<|startoflm|>", + "<|startofprev|>", + "<|nospeech|>", + "<|notimestamps|>", + "<|0.00|>", + ] + special_tokens = {} + for tok in required_specials: + special_tokens[tok] = n_vocab + n_vocab += 1 + + return tiktoken.Encoding( + name="test-encoding", + pat_str=( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| """ + r"""?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ), + mergeable_ranks=mergeable_ranks, + special_tokens=special_tokens, + ) + + +def _build_test_tokenizer() -> Tokenizer: + return Tokenizer(encoding=_build_test_encoding(), num_languages=1) + + +class TestTokenizerEncodeSpecialTokens(unittest.TestCase): + def setUp(self): + self.tokenizer = _build_test_tokenizer() + self.no_id = self.tokenizer.special_tokens["<|no|>"] + + def test_plain_text_encodes(self): + ids = self.tokenizer.encode("hello world") + self.assertIsInstance(ids, list) + self.assertTrue(ids) + + def test_language_tag_in_text_does_not_raise(self): + # The exact failure from issue #3110: ASR output containing "<|no|>" + # used to crash with "disallowed special token '<|no|>'". + ids = self.tokenizer.encode("hello <|no|> world") + self.assertIsInstance(ids, list) + self.assertTrue(ids) + + def test_nospeech_tag_in_text_does_not_raise(self): + ids = self.tokenizer.encode("<|nospeech|> something") + self.assertIsInstance(ids, list) + self.assertTrue(ids) + + def test_language_tag_alone_does_not_raise(self): + ids = self.tokenizer.encode("<|no|>") + self.assertIsInstance(ids, list) + self.assertTrue(ids) + + def test_caller_disallowed_special_is_respected(self): + # A caller that explicitly wants the strict behaviour must still get + # it (setdefault must not override an explicit kwargs). + with self.assertRaises(ValueError): + self.tokenizer.encode("<|no|>", disallowed_special="all") + + def test_allowed_special_all_keeps_special_token_ids(self): + # funasr/models/sense_voice/whisper_lib/decoding.py calls + # tokenizer.encode(prompt, allowed_special="all"); the special token + # must still be encoded as its token id, not as raw bytes. + ids = self.tokenizer.encode("<|no|>", allowed_special="all") + self.assertEqual(ids, [self.no_id]) + + +if __name__ == "__main__": + unittest.main()