Skip to content

fix: harden runtime FST and JNI handling #377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Jinleen wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: harden runtime FST and JNI handling #377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 69 additions & 9 deletions runtime/bindings/android/wetextprocessing.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
// limitations under the License.
#include <jni.h>

#include <exception>
#include <string>

#include "processor/wetext_processor.h"
#include "utils/wetext_flags.h"
#include "utils/wetext_string.h"
Expand All @@ -22,28 +25,85 @@ namespace wetextprocessing {
std::shared_ptr<wetext::Processor> processorTN;
std::shared_ptr<wetext::Processor> processorITN;

namespace {
class UtfStringChars {
public:
UtfStringChars(JNIEnv* env, jstring string)
: env_(env),
string_(string),
chars_(env->GetStringUTFChars(string, nullptr)) {}

UtfStringChars(const UtfStringChars&) = delete;
UtfStringChars& operator=(const UtfStringChars&) = delete;

~UtfStringChars() {
if (chars_ != nullptr) {
env_->ReleaseStringUTFChars(string_, chars_);
}
}

const char* get() const { return chars_; }

private:
JNIEnv* env_;
jstring string_;
const char* chars_;
};

bool CopyJString(JNIEnv* env, jstring string, std::string* output) {
UtfStringChars chars(env, string);
if (chars.get() == nullptr) {
return false;
}
*output = chars.get();
return true;
}

void ThrowIllegalState(JNIEnv* env, const char* message) {
jclass exception_class = env->FindClass("java/lang/IllegalStateException");
if (exception_class != nullptr) {
env->ThrowNew(exception_class, message);
}
}
}

void init(JNIEnv* env, jobject, jstring jModelDir) {
const char* pModelDir = env->GetStringUTFChars(jModelDir, nullptr);
std::string model_dir;
if (!CopyJString(env, jModelDir, &model_dir)) {
return;
}

std::string tnTagger = std::string(pModelDir) + "/zh_tn_tagger.fst";
std::string tnVerbalizer = std::string(pModelDir) + "/zh_tn_verbalizer.fst";
processorTN = std::make_shared<wetext::Processor>(tnTagger, tnVerbalizer);
try {
std::string tnTagger = model_dir + "/zh_tn_tagger.fst";
std::string tnVerbalizer = model_dir + "/zh_tn_verbalizer.fst";
processorTN = std::make_shared<wetext::Processor>(tnTagger, tnVerbalizer);

std::string itnTagger = std::string(pModelDir) + "/zh_itn_tagger.fst";
std::string itnVerbalizer = std::string(pModelDir) + "/zh_itn_verbalizer.fst";
processorITN = std::make_shared<wetext::Processor>(itnTagger, itnVerbalizer);
std::string itnTagger = model_dir + "/zh_itn_tagger.fst";
std::string itnVerbalizer = model_dir + "/zh_itn_verbalizer.fst";
processorITN = std::make_shared<wetext::Processor>(itnTagger, itnVerbalizer);
} catch (const std::exception& error) {
processorTN.reset();
processorITN.reset();
ThrowIllegalState(env, error.what());
}
}

jstring normalize(JNIEnv* env, jobject, jstring input) {
std::string input_text = std::string(env->GetStringUTFChars(input, nullptr));
std::string input_text;
if (!CopyJString(env, input, &input_text)) {

@pengzhendong pengzhendong Jul 28, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Please guard processorTN before the later dereference. init now resets both processors when loading fails, so calling normalize before initialization or after a handled initialization failure still reaches a native null dereference. Throw IllegalStateException and return nullptr instead.

return nullptr;
}
std::string tagged_text = processorTN->Tag(input_text);
std::string normalized_text = processorTN->Verbalize(tagged_text);

return env->NewStringUTF(normalized_text.c_str());
}

jstring inverse_normalize(JNIEnv* env, jobject, jstring input) {
std::string input_text = std::string(env->GetStringUTFChars(input, nullptr));
std::string input_text;
if (!CopyJString(env, input, &input_text)) {

@pengzhendong pengzhendong Jul 28, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Please put the complete body of both processing JNI methods inside a C++ exception boundary. Tag/Verbalize can throw (including malformed-token errors), and even the string copy may throw during allocation; letting those exceptions escape the JNI frame can terminate the app. Translate them to a Java exception and return nullptr.

return nullptr;
}
std::string tagged_text = processorITN->Tag(input_text);
std::string normalized_text = processorITN->Verbalize(tagged_text);

Expand Down
9 changes: 9 additions & 0 deletions runtime/processor/wetext_processor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,22 @@

#include "processor/wetext_processor.h"

#include <stdexcept>

#include "utils/wetext_log.h"

namespace wetext {
Processor::Processor(const std::string& tagger_path,
const std::string& verbalizer_path) {
tagger_.reset(StdVectorFst::Read(tagger_path));
if (tagger_ == nullptr) {
throw std::runtime_error("Failed to load tagger FST: " + tagger_path);
}
verbalizer_.reset(StdVectorFst::Read(verbalizer_path));
if (verbalizer_ == nullptr) {
throw std::runtime_error("Failed to load verbalizer FST: " +
verbalizer_path);
}
compiler_ = std::make_shared<StringCompiler<StdArc>>();
printer_ = std::make_shared<StringPrinter<StdArc>>();

Expand Down
8 changes: 8 additions & 0 deletions runtime/test/processor_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <stdexcept>
#include <string>
#include <vector>

Expand Down Expand Up @@ -72,6 +73,13 @@ TEST_P(ProcessorTest, NormalizeTest) {
EXPECT_EQ(processor->Normalize(written), spoken);
}

TEST(ProcessorLoadTest, ThrowsWhenFstFilesCannotBeLoaded) {
EXPECT_THROW(
wetext::Processor("/tmp/zh_tn_missing_tagger.fst",
"/tmp/zh_tn_missing_verbalizer.fst"),

@pengzhendong pengzhendong Jul 28, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] This case throws on the missing tagger, so it never exercises the new verbalizer null check. Please add a valid-tagger + missing-verbalizer case. It would also be better to use a portable unique temporary path instead of fixed /tmp names.

std::runtime_error);
}

std::vector<std::pair<std::string, std::string>> test_cases =
ParseTestCase(WETEXT_TN_DIR "/chinese/test/data/normalizer.txt");
INSTANTIATE_TEST_SUITE_P(NormalizeTest, ProcessorTest,
Expand Down
Loading