From 1e7c7d864a91076c31f75ef11ada1dd1773de9de Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Tue, 14 Jul 2026 16:16:27 -0700 Subject: [PATCH] [rebaseline_tests] Improve `--update-emsdk` and binaryen version check When running `tools/maint/rebaseline_tests.py --update-emsdk`, check both the LLVM version and Binaryen version in the currently installed SDK (`get_clang_version` and `get_binaryen_version`), and automatically patch `EXPECTED_LLVM_VERSION` in `tools/shared.py` and `EXPECTED_BINARYEN_VERSION` in `tools/building.py` if they differ. Also update `check_binaryen` in `tools/building.py` so it has the same buildbot handling (`BUILDBOT_BUILDNUMBER`) as `check_llvm_version`, only allowing `EXPECTED_BINARYEN_VERSION + 1` in CI environments. We should probably split up this script since its doing a lot more than rebaselining tests at this point, but I'll leave that for a followup. --- test/test_sanity.py | 15 ++++++++- tools/building.py | 14 ++++++--- tools/maint/rebaseline_tests.py | 55 ++++++++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/test/test_sanity.py b/test/test_sanity.py index add24e67d2be7..b44aadfb5e9f5 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -26,7 +26,7 @@ ) from decorators import no_windows, parameterized, with_env_modify -from tools import cache, ports, response_file, shared, utils +from tools import building, cache, ports, response_file, shared, utils from tools.config import EM_CONFIG from tools.shared import EMCC, config from tools.utils import delete_dir, delete_file @@ -284,6 +284,7 @@ def test_llvm(self): make_fake_tool(self.in_dir('fake', 'llvm-ar'), '%s.%s' % (expected_x, expected_y)) make_fake_tool(self.in_dir('fake', 'llvm-nm'), '%s.%s' % (expected_x, expected_y)) expect_warning = inc_x != 0 + # We have a special exception for the emscripten-release buildbot where we also allow EXPECTED_LLVM_VERSION + 1 if 'BUILDBOT_BUILDNUMBER' in os.environ and inc_x == 1: expect_warning = False if expect_warning: @@ -817,6 +818,18 @@ def test_binaryen_version(self): make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), '70') self.check_working([EMCC, test_file('hello_world.c'), '-O2'], 'unexpected binaryen version: 70 (expected ') + make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), str(building.EXPECTED_BINARYEN_VERSION)) + output = self.do([EMCC, test_file('hello_world.c'), '-O2']) + self.assertNotContained('unexpected binaryen version', output) + + # We have a special exception for the emscripten-release buildbot where we also allow EXPECTED_BINARYEN_VERSION + 1 + make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), str(building.EXPECTED_BINARYEN_VERSION + 1)) + if 'BUILDBOT_BUILDNUMBER' in os.environ: + output = self.do([EMCC, test_file('hello_world.c'), '-O2']) + self.assertNotContained('unexpected binaryen version', output) + else: + self.check_working([EMCC, test_file('hello_world.c'), '-O2'], 'unexpected binaryen version') + def test_bootstrap(self): restore_and_set_up() self.run_process([EMCC, test_file('hello_world.c')]) diff --git a/tools/building.py b/tools/building.py index 328af4bfcf1c8..6909f87fbc57c 100644 --- a/tools/building.py +++ b/tools/building.py @@ -1223,10 +1223,16 @@ def check_binaryen(bindir): except (IndexError, ValueError): exit_with_error(f'error parsing binaryen version ({output}). Please check your binaryen installation') - # Allow the expected version or the following one in order avoid needing to update both - # emscripten and binaryen in lock step in emscripten-releases. - if version not in {EXPECTED_BINARYEN_VERSION, EXPECTED_BINARYEN_VERSION + 1}: - diagnostics.warning('version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION) + if version == EXPECTED_BINARYEN_VERSION: + return True + # When running in CI environment we also silently allow the next major + # version of binaryen here so that new versions of binaryen can be rolled in + # without disruption. + if 'BUILDBOT_BUILDNUMBER' in os.environ: + if version == EXPECTED_BINARYEN_VERSION + 1: + return True + diagnostics.warning('version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION) + return False def get_binaryen_bin(): diff --git a/tools/maint/rebaseline_tests.py b/tools/maint/rebaseline_tests.py index b8df708f85811..ec57658d0c667 100755 --- a/tools/maint/rebaseline_tests.py +++ b/tools/maint/rebaseline_tests.py @@ -13,6 +13,7 @@ import argparse import json import os +import re import statistics import subprocess import sys @@ -22,7 +23,7 @@ root_dir = os.path.dirname(os.path.dirname(script_dir)) sys.path.insert(0, root_dir) -from tools import utils +from tools import building, shared, utils def run(cmd, **args): @@ -43,9 +44,7 @@ def read_size_from_json(content): def get_installed_emsdk_sha(): - emsdk = os.environ.get('EMSDK') - if not emsdk: - return None + emsdk = os.environ.get('EMSDK', os.path.join(os.path.dirname(root_dir), 'emsdk')) version_file = os.path.join(emsdk, 'upstream', '.emsdk_version') if not os.path.exists(version_file): return None @@ -107,13 +106,52 @@ def process_changed_file(filename): return f'{filename}: {old_size} => {size} [{delta:+} bytes / {percent_delta:+.2f}%]\n' +def patch_file(file_path, pattern, replacement): + """Patch a file using regular expressions.""" + content = utils.read_file(file_path) + new_content, count = re.subn(pattern, replacement, content, flags=re.MULTILINE) + if count != 1: + utils.exit_with_error(f'failed to patch {file_path} (pattern: `{pattern}`, got {count} matches, expected 1)') + utils.write_file(file_path, new_content) + + +def update_expected_llvm_version(): + clang_version_str = shared.get_clang_version() + if not clang_version_str: + utils.exit_with_error('--update-emsdk specified but could not detect clang version') + new_version = int(clang_version_str.split('.')[0]) + if new_version == shared.EXPECTED_LLVM_VERSION: + return None + print(f'update EXPECTED_LLVM_VERSION: {shared.EXPECTED_LLVM_VERSION} => {new_version}') + shared_py_file = os.path.join(root_dir, 'tools', 'shared.py') + patch_file(shared_py_file, r'^EXPECTED_LLVM_VERSION = \d+$', f'EXPECTED_LLVM_VERSION = {new_version}') + return f'Expected LLVM version updated: {shared.EXPECTED_LLVM_VERSION} => {new_version}' + + +def update_expected_binaryen_version(): + bindir = building.get_binaryen_bin() + binaryen_version_str = building.get_binaryen_version(bindir) + if not binaryen_version_str: + utils.exit_with_error('--update-emsdk specified but could not detect binaryen version') + try: + new_version = int(binaryen_version_str.splitlines()[0].split()[2]) + except (IndexError, ValueError): + utils.exit_with_error(f'error parsing binaryen version ({binaryen_version_str})') + if new_version == building.EXPECTED_BINARYEN_VERSION: + return None + print(f'update EXPECTED_BINARYEN_VERSION: {building.EXPECTED_BINARYEN_VERSION} => {new_version}') + building_py_file = os.path.join(root_dir, 'tools', 'building.py') + patch_file(building_py_file, r'^EXPECTED_BINARYEN_VERSION = \d+$', f'EXPECTED_BINARYEN_VERSION = {new_version}') + return f'Expected Binaryen version updated: {building.EXPECTED_BINARYEN_VERSION} => {new_version}' + + def main(): parser = argparse.ArgumentParser() parser.add_argument('-s', '--skip-tests', action='store_true', help="Don't actually run the tests, just analyze the existing results") parser.add_argument('-b', '--new-branch', action='store_true', help='Create a new branch containing the updates') parser.add_argument('-c', '--clear-cache', action='store_true', help='Clear the cache before rebaselining (useful when working with llvm changes)') parser.add_argument('-n', '--check-only', dest='check_only', action='store_true', help='Return non-zero if test expectations are out of date, and skip creating a git commit') - parser.add_argument('--update-emsdk', action='store_true', help='Update test/emsdk_version.txt to match the currently installed emsdk version') + parser.add_argument('--update-emsdk', action='store_true', help='Update test/emsdk_version.txt and expected tool versions to match the currently installed emsdk version') args = parser.parse_args() if args.clear_cache: @@ -121,6 +159,7 @@ def main(): current_sha = None installed_sha = None + tool_updates = [] if not args.skip_tests: if not args.check_only and run(['git', 'status', '-uno', '--porcelain']).strip(): print('tree is not clean') @@ -137,6 +176,10 @@ def main(): return 0 print(f'update emsdk: {current_sha} => {installed_sha}') utils.write_file(emsdk_version_file, installed_sha + '\n') + if update := update_expected_llvm_version(): + tool_updates.append(update) + if update := update_expected_binaryen_version(): + tool_updates.append(update) elif installed_sha and installed_sha != current_sha: utils.exit_with_error(f'installed emsdk version ({installed_sha}) does not match test/emsdk_version.txt ({current_sha}). Pass --update-emsdk to update it.') @@ -165,6 +208,8 @@ def main(): This is an automatic change generated by tools/maint/rebaseline_tests.py --update-emsdk. ''' + if tool_updates: + message += '\n'.join(tool_updates) + '\n\n' message += format_emsdk_version_update(current_sha, installed_sha) + '\n' else: message = '''Automatic rebaseline of codesize expectations. NFC