Improve rebaseline_tests.py --update-emsdk to enable llvm and binaryen version bumps

test/test_sanity.py

@@ -26,7 +26,7 @@
 )
 from decorators import no_windows, parameterized, with_env_modify
 
-from tools import cache, ports, response_file, shared, utils
+from tools import building, cache, ports, response_file, shared, utils
 from tools.config import EM_CONFIG
 from tools.shared import EMCC, config
 from tools.utils import delete_dir, delete_file
@@ -284,6 +284,7 @@ def test_llvm(self):
         make_fake_tool(self.in_dir('fake', 'llvm-ar'), '%s.%s' % (expected_x, expected_y))
         make_fake_tool(self.in_dir('fake', 'llvm-nm'), '%s.%s' % (expected_x, expected_y))
         expect_warning = inc_x != 0
+        # We have a special exception for the emscripten-release buildbot where we also allow EXPECTED_LLVM_VERSION + 1
         if 'BUILDBOT_BUILDNUMBER' in os.environ and inc_x == 1:
           expect_warning = False
         if expect_warning:
@@ -817,6 +818,18 @@ def test_binaryen_version(self):
     make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), '70')
     self.check_working([EMCC, test_file('hello_world.c'), '-O2'], 'unexpected binaryen version: 70 (expected ')
 
+    make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), str(building.EXPECTED_BINARYEN_VERSION))
+    output = self.do([EMCC, test_file('hello_world.c'), '-O2'])
+    self.assertNotContained('unexpected binaryen version', output)
+
+    # We have a special exception for the emscripten-release buildbot where we also allow EXPECTED_BINARYEN_VERSION + 1
+    make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), str(building.EXPECTED_BINARYEN_VERSION + 1))
+    if 'BUILDBOT_BUILDNUMBER' in os.environ:
+      output = self.do([EMCC, test_file('hello_world.c'), '-O2'])
+      self.assertNotContained('unexpected binaryen version', output)
+    else:
+      self.check_working([EMCC, test_file('hello_world.c'), '-O2'], 'unexpected binaryen version')
+
   def test_bootstrap(self):
     restore_and_set_up()
     self.run_process([EMCC, test_file('hello_world.c')])

tools/building.py

@@ -1223,10 +1223,16 @@ def check_binaryen(bindir):
   except (IndexError, ValueError):
     exit_with_error(f'error parsing binaryen version ({output}). Please check your binaryen installation')
 
-  # Allow the expected version or the following one in order avoid needing to update both
-  # emscripten and binaryen in lock step in emscripten-releases.
-  if version not in {EXPECTED_BINARYEN_VERSION, EXPECTED_BINARYEN_VERSION + 1}:
-    diagnostics.warning('version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION)
+  if version == EXPECTED_BINARYEN_VERSION:
+    return True
+  # When running in CI environment we also silently allow the next major
+  # version of binaryen here so that new versions of binaryen can be rolled in
+  # without disruption.
+  if 'BUILDBOT_BUILDNUMBER' in os.environ:
+    if version == EXPECTED_BINARYEN_VERSION + 1:
+      return True
+  diagnostics.warning('version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION)
+  return False
 
 
 def get_binaryen_bin():

tools/maint/rebaseline_tests.py

@@ -13,6 +13,7 @@
 import argparse
 import json
 import os
+import re
 import statistics
 import subprocess
 import sys
@@ -22,7 +23,7 @@
 root_dir = os.path.dirname(os.path.dirname(script_dir))
 
 sys.path.insert(0, root_dir)
-from tools import utils
+from tools import building, shared, utils
 
 
 def run(cmd, **args):
@@ -43,9 +44,7 @@ def read_size_from_json(content):
 
 
 def get_installed_emsdk_sha():
-  emsdk = os.environ.get('EMSDK')
-  if not emsdk:
-    return None
+  emsdk = os.environ.get('EMSDK', os.path.join(os.path.dirname(root_dir), 'emsdk'))
   version_file = os.path.join(emsdk, 'upstream', '.emsdk_version')
   if not os.path.exists(version_file):
     return None
@@ -107,20 +106,60 @@ def process_changed_file(filename):
   return f'{filename}: {old_size} => {size} [{delta:+} bytes / {percent_delta:+.2f}%]\n'
 
 
+def patch_file(file_path, pattern, replacement):
+  """Patch a file using regular expressions."""
+  content = utils.read_file(file_path)
+  new_content, count = re.subn(pattern, replacement, content, flags=re.MULTILINE)
+  if count != 1:
+    utils.exit_with_error(f'failed to patch {file_path} (pattern: `{pattern}`, got {count} matches, expected 1)')
+  utils.write_file(file_path, new_content)
+
+
+def update_expected_llvm_version():
+  clang_version_str = shared.get_clang_version()
+  if not clang_version_str:
+    utils.exit_with_error('--update-emsdk specified but could not detect clang version')
+  new_version = int(clang_version_str.split('.')[0])
+  if new_version == shared.EXPECTED_LLVM_VERSION:
+    return None
+  print(f'update EXPECTED_LLVM_VERSION: {shared.EXPECTED_LLVM_VERSION} => {new_version}')
+  shared_py_file = os.path.join(root_dir, 'tools', 'shared.py')
+  patch_file(shared_py_file, r'^EXPECTED_LLVM_VERSION = \d+$', f'EXPECTED_LLVM_VERSION = {new_version}')
+  return f'Expected LLVM version updated: {shared.EXPECTED_LLVM_VERSION} => {new_version}'
+
+
+def update_expected_binaryen_version():
+  bindir = building.get_binaryen_bin()
+  binaryen_version_str = building.get_binaryen_version(bindir)
+  if not binaryen_version_str:
+    utils.exit_with_error('--update-emsdk specified but could not detect binaryen version')
+  try:
+    new_version = int(binaryen_version_str.splitlines()[0].split()[2])
+  except (IndexError, ValueError):
+    utils.exit_with_error(f'error parsing binaryen version ({binaryen_version_str})')
+  if new_version == building.EXPECTED_BINARYEN_VERSION:
+    return None
+  print(f'update EXPECTED_BINARYEN_VERSION: {building.EXPECTED_BINARYEN_VERSION} => {new_version}')
+  building_py_file = os.path.join(root_dir, 'tools', 'building.py')
+  patch_file(building_py_file, r'^EXPECTED_BINARYEN_VERSION = \d+$', f'EXPECTED_BINARYEN_VERSION = {new_version}')
+  return f'Expected Binaryen version updated: {building.EXPECTED_BINARYEN_VERSION} => {new_version}'
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('-s', '--skip-tests', action='store_true', help="Don't actually run the tests, just analyze the existing results")
   parser.add_argument('-b', '--new-branch', action='store_true', help='Create a new branch containing the updates')
   parser.add_argument('-c', '--clear-cache', action='store_true', help='Clear the cache before rebaselining (useful when working with llvm changes)')
   parser.add_argument('-n', '--check-only', dest='check_only', action='store_true', help='Return non-zero if test expectations are out of date, and skip creating a git commit')
-  parser.add_argument('--update-emsdk', action='store_true', help='Update test/emsdk_version.txt to match the currently installed emsdk version')
+  parser.add_argument('--update-emsdk', action='store_true', help='Update test/emsdk_version.txt and expected tool versions to match the currently installed emsdk version')
   args = parser.parse_args()
 
   if args.clear_cache:
     run(['./emcc', '--clear-cache'])
 
   current_sha = None
   installed_sha = None
+  tool_updates = []
   if not args.skip_tests:
     if not args.check_only and run(['git', 'status', '-uno', '--porcelain']).strip():
       print('tree is not clean')
@@ -137,6 +176,10 @@ def main():
         return 0
       print(f'update emsdk: {current_sha} => {installed_sha}')
       utils.write_file(emsdk_version_file, installed_sha + '\n')
+      if update := update_expected_llvm_version():
+        tool_updates.append(update)
+      if update := update_expected_binaryen_version():
+        tool_updates.append(update)
     elif installed_sha and installed_sha != current_sha:
       utils.exit_with_error(f'installed emsdk version ({installed_sha}) does not match test/emsdk_version.txt ({current_sha}). Pass --update-emsdk to update it.')
 
@@ -165,6 +208,8 @@ def main():
 This is an automatic change generated by tools/maint/rebaseline_tests.py --update-emsdk.
 
 '''
+    if tool_updates:
+      message += '\n'.join(tool_updates) + '\n\n'
     message += format_emsdk_version_update(current_sha, installed_sha) + '\n'
   else:
     message = '''Automatic rebaseline of codesize expectations. NFC