diff --git a/.circleci/config.yml b/.circleci/config.yml index 112e3eca1a3e8..e514efee8239b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,12 @@ commands: bootstrap: description: "bootstrap" steps: + - run: + name: Install dependencies (Linux) + command: | + if command -v apt-get >/dev/null 2>&1; then + apt-get install -q -y cmake ninja-build + fi - run: "$EMSDK_PYTHON ./bootstrap.py" pip-install: description: "pip install" @@ -1182,30 +1188,6 @@ jobs: steps: - test-sockets-chrome - build-windows-launcher: - executor: - name: win/server-2022 - shell: bash.exe -eo pipefail - steps: - - checkout - - run: - name: "build pylauncher" - shell: cmd.exe - command: .circleci\setup_vs2022.bat && cd tools\pylauncher && call build.bat - - store_artifacts: - path: tools/pylauncher/pylauncher.exe - destination: pylauncher.exe - - install-emsdk - - pip-install: - python: "$EMSDK_PYTHON" - - run: - name: "create_entry_points" - command: $EMSDK_PYTHON tools/maint/create_entry_points.py --exe-files - - run: - name: "crossplatform tests" - command: test/runner.exe core0.test_hello_world - - # windows and mac do not have separate build and test jobs, as they only run # a limited set of tests; it is simpler and faster to do it all in one job. test-windows: @@ -1233,10 +1215,6 @@ jobs: EMTEST_BROWSER: "0" steps: - checkout - - run: - name: Build launcher - command: call .circleci\setup_vs2019.bat && cd tools\pylauncher && call build.bat - shell: cmd.exe - run: name: Install packages command: | @@ -1257,9 +1235,7 @@ jobs: - upload-test-results - run: name: "check clean" - command: | - git checkout tools/pylauncher - $EMSDK_PYTHON test/check_clean.py + command: $EMSDK_PYTHON test/check_clean.py test-mac-arm64: executor: mac-arm64 @@ -1357,7 +1333,6 @@ workflows: - test-node-compat - test-windows - test-windows-browser-firefox - - build-windows-launcher - test-mac-arm64: requires: - build-linux diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe44de52d93a..0ae178418b74e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: echo "Be sure that you have installed the current emsdk version. See test/emsdk_version.txt ($(cat test/emsdk_version.txt))." exit 1 fi + - name: Check emcc_native generated settings + run: | + ./tools/emcc_native/gen_settings.py --check clang-format-diff: # This job is disabled until we can make it more precise @@ -83,22 +86,3 @@ jobs: sudo apt-get install clang-format-19 sudo update-alternatives --install /usr/bin/git-clang-format git-clang-format /usr/bin/git-clang-format-19 100 - run: tools/maint/clang-format-diff.sh origin/$GITHUB_BASE_REF - - build-pylauncher-arm64: - name: Build pylauncher (ARM64) - runs-on: windows-11-arm - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - with: - arch: arm64 - - name: Build pylauncher - run: | - cd tools\pylauncher - call build.bat arm64 - shell: cmd - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: pylauncher-arm64.exe - path: tools/pylauncher/pylauncher-arm64.exe diff --git a/Makefile b/Makefile index bf66dcae258fa..157ff19110c64 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,18 @@ install: ./tools/install.py $(DESTDIR) npm install --omit=dev --prefix $(DESTDIR) +out/build_emcc_native/native_tests emcc_native: + cmake -B out/build_emcc_native -S tools/emcc_native -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build out/build_emcc_native --config Release + cmake --install out/build_emcc_native --config Release + +test: out/build_emcc_native/native_tests + ctest --test-dir out/build_emcc_native --output-on-failure + # Create an distributable archive of emscripten suitable for use # by end users. This archive excludes node_modules as it can include native # modules which can't be safely pre-packaged. $(DISTFILE): install tar cf $@ --exclude=node_modules -C `dirname $(DESTDIR)` `basename $(DESTDIR)` -.PHONY: dist install +.PHONY: dist install emcc_native test diff --git a/README.md b/README.md index 656e778a1d936..d0f5a6ba50021 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,11 @@ There are two primary ways to install Emscripten: The easiest way to get started is by using the Emscripten SDK. Follow the instructions on the [downloads page](https://emscripten.org/docs/getting_started/downloads.html) to install it. 2. **From a Git Checkout (Manual Installation)** - If you have cloned the repository from Git, you can install the dependencies manually and then run the bootstrap script: + If you have cloned the repository from Git, run `bootstrap.py` to build the native launcher (`emcc_native`) and set up dependencies: ```bash ./bootstrap.py ``` - For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html). + Building `emcc_native` requires CMake 3.20+ and a C++20 host compiler toolchain. Alternatively, setting `EMCC_NATIVE=0` in your environment before running `./bootstrap.py` will generate legacy Python launcher scripts (e.g. `.bat` / `.ps1` files on Windows) via `./tools/maint/create_entry_points.py`. For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html). ## Using the compiler diff --git a/bootstrap.py b/bootstrap.py index 7ed6823571fd8..14f7d620d71c1 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -95,18 +95,22 @@ def run_cmd(cmd): subprocess.run(cmd, check=True, text=True, encoding='utf-8', cwd=utils.path_from_root()) +def build_emcc_native(): + build_dir = utils.path_from_root('out/build_emcc_native') + source_dir = utils.path_from_root('tools/emcc_native') + cmd = ['cmake', '-B', build_dir, source_dir, '-DCMAKE_BUILD_TYPE=Release'] + if not utils.WINDOWS and shutil.which('ninja'): + cmd.extend(['-G', 'Ninja']) + run_cmd(cmd) + run_cmd(['cmake', '--build', build_dir, '--config', 'Release']) + run_cmd(['cmake', '--install', build_dir, '--config', 'Release']) + + actions = [ ('npm packages', [ 'package.json', 'package-lock.json', ], ['npm', 'ci']), - ('create entry points', [ - 'tools/maint/create_entry_points.py', - 'tools/pylauncher/pylauncher.exe', - 'tools/maint/run_python.bat', - 'tools/maint/run_python.sh', - 'tools/maint/run_python.ps1', - ], [sys.executable, 'tools/maint/create_entry_points.py']), ('git submodules', [ 'test/third_party/posixtestsuite/', 'test/third_party/googletest', @@ -118,6 +122,28 @@ def run_cmd(cmd): ], maybe_install_hooks), ] +if os.environ.get('EMCC_NATIVE') == '0': + actions.append(('legacy entry points', [ + 'tools/maint/create_entry_points.py', + 'tools/maint/run_python.sh', + 'tools/maint/run_python.bat', + 'tools/maint/run_python.ps1', + 'tools/maint/run_python_compiler.sh', + 'tools/maint/run_python_compiler.bat', + 'tools/maint/run_python_compiler.ps1', + ], [sys.executable, utils.path_from_root('tools/maint/create_entry_points.py')])) +else: + actions.append(('build emcc_native', [ + 'tools/emcc_native/CMakeLists.txt', + 'tools/emcc_native/main.cpp', + 'tools/emcc_native/driver.cpp', + 'tools/emcc_native/driver.h', + 'tools/emcc_native/exec.cpp', + 'tools/emcc_native/exec.h', + 'tools/emcc_native/config.cpp', + 'tools/emcc_native/config.h', + ], build_emcc_native)) + def main(args): parser = argparse.ArgumentParser(description=__doc__) diff --git a/docs/design/03-native-clang-frontend.md b/docs/design/03-native-clang-frontend.md index 7970c87c18033..efe2264257d97 100644 --- a/docs/design/03-native-clang-frontend.md +++ b/docs/design/03-native-clang-frontend.md @@ -1,6 +1,6 @@ # Design Doc: Native Launcher / Clang Frontend -- **Status**: Draft +- **Status**: Phase 1 Completed - **Bug**: https://github.com/emscripten-core/emscripten/issues/26453 ## Context diff --git a/site/source/docs/building_from_source/index.rst b/site/source/docs/building_from_source/index.rst index feffe360c1eb6..e50c7a728b9e0 100644 --- a/site/source/docs/building_from_source/index.rst +++ b/site/source/docs/building_from_source/index.rst @@ -7,11 +7,18 @@ Building Emscripten from Source Building Emscripten yourself is an alternative to getting binaries using the emsdk. -Emscripten itself is written in Python and JavaScript so it does not need to be -compiled. However, after checkout you will need to run the top level -``bootstrap.py`` script before the toolchain is usable. This performs -various steps including ``npm install`` and the creation of compiler entry -points (e.g. `.bat` files on windows). +Emscripten itself is primarily written in Python and JavaScript. However, +after checkout you will need to run the top-level ``bootstrap.py`` script +before the toolchain is usable. This performs various steps including ``npm +install`` and building the native compiler frontend launcher (``emcc_native``), +which provides high-performance ``emcc`` and ``em++`` binaries. + +Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler +toolchain (which you already need for building LLVM and Binaryen). If you prefer +not to perform a native build, setting ``EMCC_NATIVE=0`` in your environment +before running ``./bootstrap.py`` instructs it to generate legacy Python launcher +scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via +``./tools/maint/create_entry_points.py``. Emscripten comes with its own versions of some C/C++ system libraries which ``emcc`` builds automatically as and when needed (in the emsdk builds, these are diff --git a/site/source/docs/building_from_source/toolchain_what_is_needed.rst b/site/source/docs/building_from_source/toolchain_what_is_needed.rst index 33575b220282e..85506a2461cc1 100644 --- a/site/source/docs/building_from_source/toolchain_what_is_needed.rst +++ b/site/source/docs/building_from_source/toolchain_what_is_needed.rst @@ -39,7 +39,9 @@ In general a complete Emscripten environment requires the following tools. First Compiler toolchain ------------------ -When building LLVM and Binaryen from source code, whether "manually" or using the SDK, you will need a *compiler toolchain*: +When building Emscripten from source (via ``bootstrap.py`` to build +``emcc_native``), or when building LLVM and Binaryen from source, you will need +a C++20 host *compiler toolchain* and CMake 3.20+: - Windows: You will need `Visual Studio `_ (2019 or above) and `cmake `_ (3.20 or above). diff --git a/site/source/docs/contributing/developers_guide.rst b/site/source/docs/contributing/developers_guide.rst index 60cdcd0109da4..ff3d9e224a317 100644 --- a/site/source/docs/contributing/developers_guide.rst +++ b/site/source/docs/contributing/developers_guide.rst @@ -16,10 +16,22 @@ interested in helping out! Setting up ========== -For contributing to core Emscripten code, such as ``emcc.py``, you don't need to -build any binaries as ``emcc.py`` is in Python, and the core JS generation is -in JavaScript. You do still need binaries for LLVM and Binaryen, which you can -get using the emsdk. +When setting up a Git checkout of Emscripten, run the top-level ``bootstrap.py`` +script to set up dependencies (such as ``npm install``) and build the native +compiler frontend launcher (``emcc_native``): + +:: + + ./bootstrap.py + +Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler +toolchain. If you prefer not to build the native launcher, setting ``EMCC_NATIVE=0`` +in your environment before running ``./bootstrap.py`` will generate legacy Python +launcher scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via +``./tools/maint/create_entry_points.py``. + +For LLVM and Binaryen binaries, you don't need to build them from source if you +are only contributing to Emscripten; you can get them using the emsdk. If you want to contribute back to Emscripten, it is recommended that you install the precise version of the emsdk binaries that are used by Emscripten CI when diff --git a/test/test_other.py b/test/test_other.py index df63abf035d05..4ed95fa96a9a3 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -12279,9 +12279,9 @@ def test_xclang_flag(self): self.run_process([EMCC, '-c', '-o', 'out.o', '-Xclang', '-include', '-Xclang', 'foo.h', test_file('hello_world.c')]) def test_emcc_size_parsing(self): - create_file('foo.h', ' ') - self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.h'], 'error: invalid byte size `X`. Valid suffixes are: kb, mb, gb, tb') - self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.h'], 'error: invalid byte size `11PB`. Valid suffixes are: kb, mb, gb, tb') + create_file('foo.c', ' ') + self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.c'], 'error: invalid byte size `X`. Valid suffixes are: kb, mb, gb, tb') + self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.c'], 'error: invalid byte size `11PB`. Valid suffixes are: kb, mb, gb, tb') def test_native_call_before_init(self): self.set_setting('ASSERTIONS') diff --git a/tools/compile.py b/tools/compile.py index 01bc6c83b13f3..e96c5d5937a12 100644 --- a/tools/compile.py +++ b/tools/compile.py @@ -16,6 +16,9 @@ get_cflags(): In addition to compiler flags this function also returns pre-processor flags. For example, include paths and macro definitions. + +NOTE: Default compiler flag construction logic here is also implemented natively +in tools/emcc_native/driver.cpp. Keep changes in sync between both places! """ import os diff --git a/tools/emcc_native/CMakeLists.txt b/tools/emcc_native/CMakeLists.txt new file mode 100644 index 0000000000000..22b56088f02c5 --- /dev/null +++ b/tools/emcc_native/CMakeLists.txt @@ -0,0 +1,129 @@ +cmake_minimum_required(VERSION 3.20) +project(emcc_native CXX) + +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/../.." CACHE PATH "Default install prefix to Emscripten root" FORCE) +endif() + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +if(MSVC) + add_compile_options(/W4 /WX /UNDEBUG) + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) +else() + add_compile_options(-Wall -Wextra -Werror -UNDEBUG) +endif() + +enable_testing() + +add_library(native_launcher_lib OBJECT + config.cpp + driver.cpp + exec.cpp +) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +# Build `emcc` native launcher +add_executable(emcc main.cpp) +target_link_libraries(emcc PRIVATE native_launcher_lib) + +set(TOP_LEVEL_ENTRY_POINTS + em++ + emar + embuilder + emcmake + em-config + emconfigure + emmake + emranlib + emrun + emscons + emsize + emprofile + emdwp + emnm + emstrip + emsymbolizer + emscan-deps + empath-split +) + +set(TOOLS_ENTRY_POINTS + file_packager + webidl_binder +) + +set(TEST_ENTRY_POINTS + runner +) + +if(WIN32) + foreach(entry ${TOP_LEVEL_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/${entry}$ + COMMENT "Creating ${entry} launcher executable" + ) + install(PROGRAMS $/${entry}$ DESTINATION .) + endforeach() + + foreach(entry ${TOOLS_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/tools + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/tools/${entry}$ + COMMENT "Creating tools/${entry} launcher executable" + ) + install(PROGRAMS $/tools/${entry}$ DESTINATION tools) + endforeach() + + foreach(entry ${TEST_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/test + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/test/${entry}$ + COMMENT "Creating test/${entry} launcher executable" + ) + install(PROGRAMS $/test/${entry}$ DESTINATION test) + endforeach() +else() + foreach(entry ${TOP_LEVEL_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink $ $/${entry}$ + COMMENT "Creating ${entry} launcher executable" + ) + install(PROGRAMS $/${entry}$ DESTINATION .) + endforeach() + + foreach(entry ${TOOLS_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/tools + COMMAND ${CMAKE_COMMAND} -E create_symlink ../$ $/tools/${entry}$ + COMMENT "Creating tools/${entry} launcher executable" + ) + install(PROGRAMS $/tools/${entry}$ DESTINATION tools) + endforeach() + + foreach(entry ${TEST_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/test + COMMAND ${CMAKE_COMMAND} -E create_symlink ../$ $/test/${entry}$ + COMMENT "Creating test/${entry} launcher executable" + ) + install(PROGRAMS $/test/${entry}$ DESTINATION test) + endforeach() +endif() + +# Build unit tests +add_executable(native_tests tests/test_native.cpp) +target_link_libraries(native_tests PRIVATE native_launcher_lib) + +add_test(NAME native_tests COMMAND native_tests) + +install(TARGETS emcc DESTINATION .) diff --git a/tools/emcc_native/README.md b/tools/emcc_native/README.md new file mode 100644 index 0000000000000..b8eb870077c2c --- /dev/null +++ b/tools/emcc_native/README.md @@ -0,0 +1,131 @@ +# Native Clang Frontend Launcher (`emcc_native`) + +`emcc_native` is a C++ launcher for Emscripten's Python-based tools. In its +basic function it acts as a replacement for the Window-only `pylauncher` and +the legacy `run_python` launcher scripts (`.sh`, `.bat`, `.ps1`). + +However, it goes one step beyond previous tools by bypassing Python completely +for many common invocations of the compiler, and directly running `clang` or +`clang++`. + +## Overview & Architecture + +When executing large builds, `emcc` can be run hundreds or thousands of times. +Launching the Python interpreter for each translation unit can add significant +overhead (especially on Windows starting Python can be more expensive). + +`emcc_native` provides native executables (`emcc`, `em++`, etc) that: +1. **Directly invoke Clang** for pure compilation steps (`-c`, `-S`, `-E`, + `-M`, `-MM`), injecting: + - Target triple (`-target wasm32-unknown-emscripten` or + `wasm64-unknown-emscripten`) + - Frontend exceptions flag (`-fignore-exceptions`) + - Default LLVM backend flags (e.g. `-mllvm -enable-emscripten-sjlj`) + - Emscripten sysroot (`--sysroot=/sysroot`) + - Clang sysroot include paths (e.g. `-Xclang -iwithsysroot/include/compat`) + - SIMD/SSE/NEON preprocessor macros (`-D__SSE__=1`, `-D__SSE2__=1`, + `-D__ARM_NEON__=1`, etc.) when architecture flags are specified + - Visibility flag (`-fvisibility=default` when `-fPIC` is passed without + `-fvisibility`) +2. **Ignore compile-unused linker flags**: Link-only flags (`--js-library`, + `--embed-file`, etc.) and linker settings (`-sEXPORTED_FUNCTIONS`, etc.) + are ignored during compilation (with diagnostic warnings matching + `emcc.py`), allowing compile steps with link flags to run natively. +3. **Fall back to Python** (`emcc.py` / `em++.py`) when link-phase invocations + are run, or when compile-time `-s` settings or system flags + (`--clear-cache`, `--build`, `--tracing`, etc.) are present. +4. Directly run Python for non-compiler tools (e.g. `embuilder`, `emsymbolizer`) + +## Building + +Building requires CMake 3.20+ and a C++20 compiler. + +Using CMake directly: + +```bash +cmake -B out/build_emcc_native -S tools/emcc_native +cmake --build out/build_emcc_native +cmake --install out/build_emcc_native +``` + +Alternatively, you can build using `bootstrap.py` or `make`: + +```bash +./bootstrap.py +# or +make emcc_native +``` + +The output executable (`emcc`, `em++`, etc) will be installed directly +alongside their python counterparts. + +## Running Tests + +To run the unit and integration tests via `ctest`: + +```bash +ctest --test-dir out/build_emcc_native --output-on-failure +``` + +Or via `make`: + +```bash +make test +``` + +## Code Generation + +Compile-time settings, link-only flags, and Emscripten warning options are +generated in `generated_settings.h`. To update this header from Python +definitions, run: + +```bash +./tools/emcc_native/gen_settings.py +``` + +To verify whether `generated_settings.h` is up to date: + +```bash +./tools/emcc_native/gen_settings.py --check +``` + +## Configuration & Environment Variables + +- `EMCC_NATIVE`: + - Set to `0` to disable the native driver and unconditionally fall back to + `emcc.py`. When set during `./bootstrap.py`, it instructs `bootstrap.py` to + generate legacy Python launcher scripts instead of building `emcc_native`. + - Set to `1` to force strict native mode; if an invocation requires falling + back to Python, `emcc_native` will print the fallback reason and exit with + an error (useful for debugging). +- `EMCC_NATIVE_DEBUG`: When set (e.g. `EMCC_NATIVE_DEBUG=1`), logs launcher + decision details (whether direct Clang execution or Python fallback was + selected, reason, target binary, and command arguments) specifically for the + native launcher without enabling Python driver debug output. +- `EMCC_DEBUG`: When set (e.g. `EMCC_DEBUG=1`), logs launcher decision details + along with Python driver debug output. +- `EMSDK_PYTHON`: Path to the Python executable (defaults to `python3` or + `python.exe` on Windows). +- `EM_CACHE`: Path to Emscripten cache directory (defaults to + `/cache`). +- `EM_CONFIG`: Path to `.emscripten` configuration file (reads `LLVM_ROOT` and + `CACHE`). +- `EM_LLVM_ROOT`: Environment variable override for the directory containing + LLVM binaries (`clang`, `clang++`). + +## CI Benchmark Results + +Compile-time performance was benchmarked on CI across Linux, macOS, and Windows +(`embuilder build libc --force` compiling 1,075 files sequentially with +`EMCC_CORES=1`, `EMCC_USE_NINJA=0`, and `EMCC_BATCH_BUILD=0`). + +| Platform | Before (Python Baseline) | After (Native Launcher) | Improvement | Speedup | +| :---------: | :----------------------: | :---------------------: | :------------------------: | :-------: | +| **Linux** | 181.96 s (169.3 ms/file) | 64.82 s (60.3 ms/file) | -117.14 s (-109.0 ms/file) | **2.81x** | +| **Windows** | 343.90 s (319.9 ms/file) | 105.12 s (97.8 ms/file) | -238.78 s (-222.1 ms/file) | **3.27x** | +| **macOS** | 162.08 s (150.6 ms/file) | 64.73 s (60.1 ms/file) | -97.35 s (-90.5 ms/file) | **2.50x** | + +As expected, because process creation and `python.exe` startup carry +significantly higher overhead on Windows than on POSIX systems, the speedup on +Windows CI (**3.27x**, saving over 222 ms per invocation) is even larger than +on Linux and macOS. diff --git a/tools/emcc_native/benchmark.py b/tools/emcc_native/benchmark.py new file mode 100755 index 0000000000000..4fda4ceed7b31 --- /dev/null +++ b/tools/emcc_native/benchmark.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +"""Benchmark Emscripten compiler invocation speed across CI platforms. + +Measures the elapsed time required to compile small source files (by default, +building libc via embuilder with batching and Ninja disabled so each C file is +invoked individually). Running this script before compiling the native launcher +benchmarks the Python driver baseline; running it after compiling benchmarks +the native C++ launcher. +""" + +import argparse +import os +import subprocess +import sys +import time + +script_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(os.path.dirname(script_dir)) +sys.path.insert(0, root_dir) + +from tools.utils import WINDOWS + + +def find_native_launcher(): + ext = '.exe' if WINDOWS else '' + native_bin = os.path.join(root_dir, 'emcc' + ext) + if os.path.exists(native_bin): + try: + with open(native_bin, 'rb') as f: + data = f.read(4) + if data.startswith((b'#!', b'@echo', b'rem')): + return None + return native_bin + except OSError: + pass + return None + + +def run_benchmark(target='libc', cores=1, iterations=1): + native_launcher = find_native_launcher() + if native_launcher: + mode = f'Native Launcher ({os.path.relpath(native_launcher, root_dir)})' + else: + mode = 'Python Launcher (baseline)' + + env = os.environ.copy() + env['EMCC_CORES'] = str(cores) + env['EMCC_USE_NINJA'] = '0' + env['EMCC_BATCH_BUILD'] = '0' + env.pop('EM_COMPILER_WRAPPER', None) + if native_launcher: + if 'EMCC_NATIVE' not in env: + env['EMCC_NATIVE'] = '1' + else: + env.pop('EMCC_NATIVE', None) + + embuilder_py = os.path.join(root_dir, 'embuilder.py') + cmd = [sys.executable, embuilder_py, 'build', target, '--force'] + + print('=' * 60) + print('Emscripten Compiler Benchmark') + print('=' * 60) + print(f'Mode: {mode}') + print(f'Target: {target}') + print(f'Iterations: {iterations}') + print(f'Settings: EMCC_CORES={cores}, EMCC_USE_NINJA=0, EMCC_BATCH_BUILD=0') + print('=' * 60) + + times = [] + for i in range(1, iterations + 1): + if iterations > 1: + print(f'\n--- Iteration {i} of {iterations} ---') + start_time = time.perf_counter() + res = subprocess.run(cmd, env=env, check=False) + elapsed = time.perf_counter() - start_time + if res.returncode != 0: + print(f'Error: benchmark command failed with exit code {res.returncode}') + return res.returncode + times.append(elapsed) + print(f'Iteration {i} took: {elapsed:.3f} s') + + print('\n' + '=' * 60) + print('Benchmark Summary') + print('=' * 60) + print(f'Mode: {mode}') + if iterations == 1: + print(f'Total Time: {times[0]:.3f} s') + else: + avg_time = sum(times) / len(times) + min_time = min(times) + max_time = max(times) + print(f'Average Time: {avg_time:.3f} s') + print(f'Min Time: {min_time:.3f} s') + print(f'Max Time: {max_time:.3f} s') + print('=' * 60) + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description='Benchmark Emscripten compiler invocation speed.', + ) + parser.add_argument( + 'target', + nargs='?', + default='libc', + help='Library target to build (default: libc)', + ) + parser.add_argument( + '--cores', + type=int, + default=1, + help='Number of cores for EMCC_CORES (default: 1)', + ) + parser.add_argument( + '-n', + '--iterations', + type=int, + default=1, + help='Number of benchmark iterations to run (default: 1)', + ) + args = parser.parse_args() + + return run_benchmark( + target=args.target, cores=args.cores, iterations=args.iterations, + ) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/emcc_native/config.cpp b/tools/emcc_native/config.cpp new file mode 100644 index 0000000000000..5593471dbe7e7 --- /dev/null +++ b/tools/emcc_native/config.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "config.h" + +#include +#include +#include +#include + +#include + +namespace fs = std::filesystem; + +namespace emscripten { + +using std::string_view; + +namespace { + +std::string get_env(const char* name) { + const char* val = std::getenv(name); + return val ? std::string(val) : std::string(); +} + +string_view trim(string_view str) { + size_t first = str.find_first_not_of(" \t\r\n"); + if (first == string_view::npos) + return ""; + size_t last = str.find_last_not_of(" \t\r\n"); + return str.substr(first, (last - first + 1)); +} + +string_view strip_quotes(string_view str) { + string_view s = trim(str); + if (s.size() >= 2 && ((s.starts_with('\'') && s.ends_with('\'')) || + (s.starts_with('"') && s.ends_with('"')))) { + return s.substr(1, s.size() - 2); + } + return s; +} + +void set_env_var(const std::string& key, const std::string& val) { +#ifdef _WIN32 + _putenv_s(key.c_str(), val.c_str()); +#else + setenv(key.c_str(), val.c_str(), 1); +#endif +} + +std::string expand_user(string_view path) { + if (path.starts_with('~')) { + std::string home = get_env("HOME"); + if (home.empty()) { + home = get_env("USERPROFILE"); + } + if (!home.empty()) { + return home + std::string(path.substr(1)); + } + } + return std::string(path); +} + +std::string expand_vars(string_view input) { + std::string s = expand_user(input); + if (s.empty()) + return s; + + std::string result; + size_t i = 0; + while (i < s.size()) { + if (s[i] == '$') { + if (i + 1 < s.size() && s[i + 1] == '{') { + size_t end = s.find('}', i + 2); + if (end != std::string::npos) { + std::string var_name = s.substr(i + 2, end - (i + 2)); + result += get_env(var_name.c_str()); + i = end + 1; + continue; + } + } else { + size_t start = i + 1; + size_t end = start; + while (end < s.size() && + (std::isalnum(static_cast(s[end])) || s[end] == '_')) { + ++end; + } + if (end > start) { + std::string var_name = s.substr(start, end - start); + result += get_env(var_name.c_str()); + i = end; + continue; + } + } + } +#ifdef _WIN32 + else if (s[i] == '%') { + size_t end = s.find('%', i + 1); + if (end != std::string::npos && end > i + 1) { + std::string var_name = s.substr(i + 1, end - (i + 1)); + result += get_env(var_name.c_str()); + i = end + 1; + continue; + } + } +#endif + + result += s[i]; + ++i; + } + + return result; +} + +} // namespace + +// Search order for the config file (must match find_config_file() in tools/config.py): +// 1. Specified via EM_CONFIG environment variable +// 2. Local .emscripten file in emscripten_root (/.emscripten) +// 3. Embedded config file two levels above emscripten_root, as used by +// `emsdk --embedded` (/../../.emscripten) +// 4. User home directory config (~/.emscripten) +fs::path find_config_file(const fs::path& emscripten_root) { + std::string env_config = get_env("EM_CONFIG"); + if (!env_config.empty() && fs::exists(env_config)) { + return fs::path(env_config); + } + + fs::path root_config = emscripten_root / ".emscripten"; + if (fs::exists(root_config)) { + return root_config; + } + + // Look two levels up for emsdk --embedded compatibility + // (e.g. emsdk/upstream/emscripten or emsdk/emscripten/x.y.z -> emsdk) + fs::path emsdk_embedded_config = + emscripten_root.parent_path().parent_path() / ".emscripten"; + if (fs::exists(emsdk_embedded_config)) { + return emsdk_embedded_config; + } + + std::string home = get_env("HOME"); + if (home.empty()) + home = get_env("USERPROFILE"); + if (!home.empty()) { + fs::path home_config = fs::path(home) / ".emscripten"; + if (fs::exists(home_config)) { + return home_config; + } + } + + return ""; +} + +Config load_config(const fs::path& emscripten_root) { + Config config; + + fs::path config_file = find_config_file(emscripten_root); + if (!config_file.empty() && fs::exists(config_file)) { + set_env_var("CFGDIR", config_file.parent_path().string()); + std::ifstream in(config_file); + std::string line; + while (std::getline(in, line)) { + size_t comment = line.find('#'); + if (comment != std::string::npos) { + line = line.substr(0, comment); + } + string_view tline = trim(line); + if (tline.empty()) + continue; + + size_t eq = tline.find('='); + if (eq != string_view::npos) { + string_view key = trim(tline.substr(0, eq)); + if (key == "LLVM_ROOT" || key == "CACHE") { + string_view raw_val = trim(tline.substr(eq + 1)); + if (raw_val.find('+') != string_view::npos || + (!raw_val.empty() && raw_val.front() != '\'' && + raw_val.front() != '"' && raw_val.front() != '$')) { + config.failure = true; + config.failure_reason = + "Complex expression in config file for " + std::string(key) + ": " + std::string(raw_val); + continue; + } + std::string val = expand_vars(strip_quotes(raw_val)); + if (key == "LLVM_ROOT") { + config.llvm_root = val; + } else if (key == "CACHE") { + config.em_cache = val; + } + } + } + } + } + + // Override with environment variables if present + std::string env_llvm = get_env("EM_LLVM_ROOT"); + if (!env_llvm.empty()) { + config.llvm_root = env_llvm; + } + + std::string env_cache = get_env("EM_CACHE"); + if (!env_cache.empty()) { + config.em_cache = env_cache; + } + + // Apply defaults + if (config.em_cache.empty()) { + config.em_cache = (emscripten_root / "cache").string(); + } + + return config; +} + +} // namespace emscripten diff --git a/tools/emcc_native/config.h b/tools/emcc_native/config.h new file mode 100644 index 0000000000000..929aac067ac57 --- /dev/null +++ b/tools/emcc_native/config.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_CONFIG_H +#define EMCC_NATIVE_CONFIG_H + +#include +#include +#include + +namespace emscripten { + +namespace fs = std::filesystem; + +struct Config { + // There are more possible settings in an emscripten config + // but the native launcher only cares about these two. + std::string llvm_root; + std::string em_cache; + bool failure = false; + std::string failure_reason; +}; + +// Find the config file (.emscripten) location. +fs::path find_config_file(const fs::path& emscripten_root); + +// Parse configuration file and environment variables. +Config load_config(const fs::path& emscripten_root); + +} // namespace emscripten + +#endif // EMCC_NATIVE_CONFIG_H diff --git a/tools/emcc_native/driver.cpp b/tools/emcc_native/driver.cpp new file mode 100644 index 0000000000000..80fb4ec55283f --- /dev/null +++ b/tools/emcc_native/driver.cpp @@ -0,0 +1,761 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "driver.h" +#include "exec.h" +#include "generated_settings.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace emscripten { + +using std::string_view; + +namespace { + +constexpr string_view get_extension(string_view path) { + size_t idx = path.rfind('.'); + if (idx == string_view::npos || idx == 0) { + return {}; + } + size_t slash = path.find_last_of("/\\"); + if (slash != string_view::npos && slash > idx) { + return {}; + } + return path.substr(idx); +} + +// Flags that require fallback to the Python driver because they represent +// complex operations or standing system commands. +// NOTE: Keep in sync with system options in tools/cmdline.py and complex +// options in tools/compile.py. +const std::unordered_set COMPLEX_OR_SYSTEM_FLAGS = { + "--clear-cache", + "--clear-ports", + "--build", + "--generate-config", + "--compiler-wrapper", + "--tracing", + "--memoryprofiler", + "--em-config", +}; + +// Subset of LINK_ONLY_FLAGS that take a value argument when passed without '='. +const std::unordered_set LINK_ONLY_FLAGS_WITH_ARGS = { + "--closure", + "--embed-file", + "--exclude-file", + "--extern-post-js", + "--extern-pre-js", + "--js-library", + "--js-transform", + "--oformat", + "--output-eol", + "--output_eol", + "--post-js", + "--pre-js", + "--preload-file", + "--shell-file", + "--source-map-base", +}; + +// Default LLVM backend arguments injected during compilation. +// NOTE: Keep in sync with llvm_backend_args() in tools/building.py. +const std::vector DEFAULT_LLVM_BACKEND_FLAGS = { + "-mllvm", + "-combiner-global-alias-analysis=false", + "-mllvm", + "-enable-emscripten-sjlj", + "-mllvm", + "-disable-lsr", +}; + +// Warning state for diagnostics +bool g_warn_unused = true; +bool g_error_unused = false; + +void parse_warning_flags(const std::vector& user_args) { + g_warn_unused = true; + g_error_unused = false; + for (string_view arg : user_args) { + if (arg == "-w") { + g_warn_unused = false; + } else if (arg == "-Werror") { + g_error_unused = true; + } else if (arg == "-Wno-error") { + g_error_unused = false; + } else if (arg == "-Wunused-command-line-argument") { + g_warn_unused = true; + } else if (arg == "-Wno-unused-command-line-argument") { + g_warn_unused = false; + } else if (arg == "-Werror=unused-command-line-argument") { + g_warn_unused = true; + g_error_unused = true; + } else if (arg == "-Wno-error=unused-command-line-argument") { + g_error_unused = false; + } + } +} + +void emit_unused_warning(string_view msg) { + if (!g_warn_unused) { + return; + } + if (g_error_unused) { + std::cerr << "emcc: error: " << msg + << " [-Wunused-command-line-argument] [-Werror]" << std::endl; + std::exit(1); + } else { + std::cerr << "emcc: warning: " << msg << " [-Wunused-command-line-argument]" + << std::endl; + } +} + +void create_python_command(const fs::path& script_path, + const std::vector& user_args, + DriverDecision& decision) { + decision.target_binary = get_python_executable(); + decision.target_args.push_back("-E"); +#ifdef _WIN32 + decision.target_args.push_back("-X"); + decision.target_args.push_back("utf8"); +#endif + decision.target_args.push_back(script_path.generic_string()); + for (const auto& arg : user_args) { + decision.target_args.push_back(arg); + } +} + +void create_compiler_fallback_command(bool is_cxx, + const fs::path& emscripten_root, + const std::vector& user_args, + DriverDecision& decision) { + fs::path script = emscripten_root / (is_cxx ? "em++.py" : "emcc.py"); + create_python_command(script, user_args, decision); +} + +// Process SIMD/SSE/NEON feature flags and inject corresponding macro +// definitions. NOTE: Keep in sync with get_cflags() in tools/compile.py and +// SIMD_INTEL_FEATURE_TOWER / SIMD_NEON_FLAGS in tools/cmdline.py. +void handle_simd_flags(const std::vector& filtered_user_args, + DriverDecision& decision) { + bool has_simd = false; + bool has_sse = false, has_sse2 = false, has_sse3 = false, has_ssse3 = false; + bool has_sse4_1 = false, has_sse4_2 = false, has_avx = false, + has_avx2 = false; + bool has_fma = false, has_neon = false; + bool has_intel_simd = false; + + for (string_view arg : filtered_user_args) { + if (arg == "-msimd128" || arg == "-mrelaxed-simd") { + has_simd = true; + } else if (arg == "-msse") { + has_sse = true; + has_intel_simd = true; + } else if (arg == "-msse2") { + has_sse = has_sse2 = true; + has_intel_simd = true; + } else if (arg == "-msse3") { + has_sse = has_sse2 = has_sse3 = true; + has_intel_simd = true; + } else if (arg == "-mssse3") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = true; + has_intel_simd = true; + } else if (arg == "-msse4.1") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = true; + has_intel_simd = true; + } else if (arg == "-msse4.2" || arg == "-msse4") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + true; + has_intel_simd = true; + } else if (arg == "-mavx") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + has_avx = true; + has_intel_simd = true; + } else if (arg == "-mavx2") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + has_avx = has_avx2 = true; + has_intel_simd = true; + } else if (arg == "-mfma") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + has_avx = has_avx2 = has_fma = true; + has_intel_simd = true; + } else if (arg == "-mfpu=neon" || arg == "-mneon") { + has_neon = true; + } + } + + if ((has_intel_simd || has_neon) && !has_simd) { + std::cerr << "emcc: error: passing any of -msse, -msse2, -msse3, -mssse3, " + "-msse4.1, -msse4.2, -msse4, -mavx, -mavx2, -mfma, -mfpu=neon " + "flags also requires passing -msimd128 (or -mrelaxed-simd)!" + << std::endl; + std::exit(1); + } + + if (has_sse || has_neon) decision.target_args.push_back("-D__SSE__=1"); + if (has_sse2) decision.target_args.push_back("-D__SSE2__=1"); + if (has_sse3) decision.target_args.push_back("-D__SSE3__=1"); + if (has_ssse3) decision.target_args.push_back("-D__SSSE3__=1"); + if (has_sse4_1) decision.target_args.push_back("-D__SSE4_1__=1"); + if (has_sse4_2) decision.target_args.push_back("-D__SSE4_2__=1"); + if (has_avx) decision.target_args.push_back("-D__AVX__=1"); + if (has_avx2) decision.target_args.push_back("-D__AVX2__=1"); + if (has_fma) decision.target_args.push_back("-D__FMA__=1"); + if (has_neon) decision.target_args.push_back("-D__ARM_NEON__=1"); +} + +// Construct the native Clang/Clang++ binary path and compiler argument vector. +// NOTE: Keep in sync with get_clang_flags() and get_cflags() in +// tools/compile.py. +void create_clang_command(bool is_cxx, + bool is_wasm64, + bool is_asm_only, + const std::vector& filtered_user_args, + const Config& config, + DriverDecision& decision) { + std::string clang_name = is_cxx ? "clang++" : "clang"; +#ifdef _WIN32 + clang_name += ".exe"; +#endif + + if (!config.llvm_root.empty()) { + decision.target_binary = (fs::path(config.llvm_root) / clang_name).generic_string(); + } else { + decision.target_binary = clang_name; + } + + // Target flags + std::string target_triple = + is_wasm64 ? "wasm64-unknown-emscripten" : "wasm32-unknown-emscripten"; + decision.target_args.push_back("-target"); + decision.target_args.push_back(target_triple); + + if (!is_asm_only) { + // Frontend exceptions flag + bool has_exceptions = false; + for (string_view arg : filtered_user_args) { + if (arg == "-fexceptions" || arg == "-fwasm-exceptions" || + arg == "-fno-ignore-exceptions") { + has_exceptions = true; + break; + } + } + if (!has_exceptions) { + decision.target_args.push_back("-fignore-exceptions"); + } else { + decision.target_args.push_back("-mllvm"); + decision.target_args.push_back("-enable-emscripten-cxx-exceptions"); + } + + // Backend flags + for (const auto& flag : DEFAULT_LLVM_BACKEND_FLAGS) { + decision.target_args.push_back(flag); + } + + // Sysroot + fs::path sysroot = fs::path(config.em_cache) / "sysroot"; + decision.target_args.push_back("--sysroot=" + sysroot.generic_string()); + + // Handle SIMD flags + handle_simd_flags(filtered_user_args, decision); + + // Check user args for special flags + bool nostdinc = false; + bool has_fpic = false; + bool has_fvisibility = false; + bool has_pthread = false; + bool has_explicit_pthread = false; + + for (string_view arg : filtered_user_args) { + if (arg == "-nostdinc") { + nostdinc = true; + } else if (arg == "-fPIC") { + has_fpic = true; + } else if (arg.starts_with("-fvisibility")) { + has_fvisibility = true; + } else if (arg == "-pthread") { + has_pthread = true; + has_explicit_pthread = true; + } else if (arg == "-fopenmp" || arg == "-fopenmp=libomp") { + has_pthread = true; + } + } + + if (has_pthread) { + decision.target_args.push_back("-D__EMSCRIPTEN_SHARED_MEMORY__=1"); + if (!has_explicit_pthread) { + decision.target_args.push_back("-pthread"); + } + } + + if (has_fpic && !has_fvisibility) { + decision.target_args.push_back("-fvisibility=default"); + } + + if (!nostdinc) { + decision.target_args.push_back("-Xclang"); + decision.target_args.push_back("-iwithsysroot/include/fakesdl"); + decision.target_args.push_back("-Xclang"); + decision.target_args.push_back("-iwithsysroot/include/compat"); + } + } + + for (string_view arg : filtered_user_args) { + decision.target_args.push_back(std::string(arg)); + } +} + +bool is_upper_identifier(string_view s) { + if (s.empty() || std::isdigit(static_cast(s[0]))) { + return false; + } + bool has_upper = false; + for (char c : s) { + unsigned char uc = static_cast(c); + if (std::isupper(uc)) { + has_upper = true; + } else if (!std::isdigit(uc) && uc != '_') { + return false; + } + } + return has_upper; +} + +bool is_dash_s_setting(const std::vector& user_args, + size_t i, + string_view& setting_key, + bool& ate_next) { + string_view arg = user_args[i]; + ate_next = false; + string_view val; + if (arg == "-s") { + if (i + 1 >= user_args.size()) + return false; + val = user_args[i + 1]; + ate_next = true; + } else if (arg.starts_with("-s")) { + val = arg.substr(2); + } else { + return false; + } + + size_t eq = val.find('='); + if (eq != string_view::npos) { + setting_key = val.substr(0, eq); + } else { + setting_key = val; + } + return is_upper_identifier(setting_key); +} + +bool is_emscripten_only_warning(string_view arg) { + if (!arg.starts_with("-W")) { + return false; + } + string_view name = arg.substr(2); + if (name.starts_with("error=")) { + name = name.substr(6); + } else if (name.starts_with("no-error=")) { + name = name.substr(9); + } else if (name.starts_with("no-")) { + name = name.substr(3); + } + return EMSCRIPTEN_ONLY_WARNINGS.contains(name); +} + +} // namespace + +std::string get_python_executable() { + const char* env_python = std::getenv("EMSDK_PYTHON"); + if (env_python && env_python[0] != '\0') { + return env_python; + } +#ifdef _WIN32 + return "python.exe"; +#else + return "python3"; +#endif +} + +bool is_assembly_only(const std::vector& user_args) { + static const std::unordered_set ASM_EXTS = {".s", ".S"}; + static const std::unordered_set C_EXTS = { + ".c", ".i", ".cppm", ".pcm", ".cpp", ".cxx", ".cc", ".c++", + ".CPP", ".CXX", ".C", ".CC", ".C++", ".ii", ".m", ".mi", ".mm", ".mii", + ".bc", ".ll" + }; + + bool has_asm = false; + bool has_c_source = false; + + for (size_t i = 0; i < user_args.size(); ++i) { + const std::string& arg = user_args[i]; + if (arg.empty() || arg[0] == '-') { + if ((arg == "-o" || arg == "-I" || arg == "-L" || arg == "-include" || + arg == "-isystem" || arg == "-MF" || arg == "-MT" || arg == "-MQ" || + arg == "-x") && i + 1 < user_args.size()) { + ++i; + } + continue; + } + string_view ext = get_extension(arg); + if (ASM_EXTS.contains(ext)) { + has_asm = true; + } else if (C_EXTS.contains(ext)) { + has_c_source = true; + } + } + + return has_asm && !has_c_source; +} + +// Check if any input argument is a header file (via extension) or if an explicit +// header language flag (e.g. -xc++-header) is specified. Compiling header inputs +// is a compile-only operation that generates precompiled headers (.pch / .gch). +// NOTE: Keep in sync with HEADER_EXTENSIONS and phase_setup() in emcc.py. +bool has_header_inputs(const std::vector& user_args) { + static const std::unordered_set HEADER_EXTS = { + ".h", ".hxx", ".hpp", ".hh", ".H", ".HXX", ".HPP", ".HH" + }; + + for (size_t i = 0; i < user_args.size(); ++i) { + const std::string& arg = user_args[i]; + if (arg.empty()) { + continue; + } + + if (arg == "-x") { + if (i + 1 < user_args.size() && user_args[i + 1].find("header") != std::string::npos) { + return true; + } + if (i + 1 < user_args.size()) { + ++i; + } + continue; + } + if (arg.starts_with("-x") && arg.find("header") != std::string::npos) { + return true; + } + + if (arg[0] == '-') { + if ((arg == "-o" || arg == "-I" || arg == "-L" || arg == "-include" || + arg == "-isystem" || arg == "-MF" || arg == "-MT" || arg == "-MQ") && + i + 1 < user_args.size()) { + ++i; + } + continue; + } + + std::string_view ext = get_extension(arg); + if (HEADER_EXTS.contains(ext)) { + return true; + } + } + + return false; +} + +static std::string get_tool_name(std::string_view driver_arg0) { + if (driver_arg0.empty()) { + std::cerr << "emcc_native: error: empty command name (argv[0])" << std::endl; + std::exit(1); + } + std::string tool = fs::path(driver_arg0).stem().string(); + if (tool.empty()) { + std::cerr << "emcc_native: error: unable to determine tool name from command '" + << driver_arg0 << "'" << std::endl; + std::exit(1); + } + return tool; +} + +static fs::path find_script_for_tool(const fs::path& emscripten_root, + const fs::path& exe_path, + string_view tool) { + static const char* const SEARCH_SUBDIRS[] = {"", "tools", "test"}; + + for (const char* subdir : SEARCH_SUBDIRS) { + fs::path candidate = emscripten_root / subdir / (std::string(tool) + ".py"); + if (fs::exists(candidate)) { + return candidate; + } + } + + if (exe_path.has_parent_path()) { + fs::path exe_dir = exe_path.parent_path(); + for (const char* subdir : SEARCH_SUBDIRS) { + fs::path candidate = exe_dir / subdir / (std::string(tool) + ".py"); + if (fs::exists(candidate)) { + return candidate; + } + } + } + + std::cerr << "emcc_native: error: python script for tool '" << tool << "' not found" << std::endl; + std::exit(1); +} + +// Analyze user arguments to determine if native compilation is supported or if +// fallback to Python is required. +DriverDecision analyze_request(string_view driver_arg0, + const fs::path& emscripten_root, + const fs::path& exe_path, + const std::vector& user_args, + const Config& config) { + DriverDecision decision; + std::string tool = get_tool_name(driver_arg0); + bool is_compiler = (tool == "emcc" || tool == "em++"); + + if (!is_compiler) { + fs::path script_path = find_script_for_tool(emscripten_root, exe_path, tool); + decision.use_fallback = true; + decision.reason = "Tool " + tool + " runs via Python script (" + script_path.filename().string() + ")"; + create_python_command(script_path, user_args, decision); + return decision; + } + + bool is_cxx = (tool == "em++"); + + // Environment variable override to disable native driver + const char* native_env = std::getenv("EMCC_NATIVE"); + if (native_env && std::string(native_env) == "0") { + decision.use_fallback = true; + decision.reason = "EMCC_NATIVE set to disable native launcher"; + } + + const char* compiler_wrapper = std::getenv("EM_COMPILER_WRAPPER"); + if (compiler_wrapper && compiler_wrapper[0] != '\0') { + decision.use_fallback = true; + decision.reason = "EM_COMPILER_WRAPPER configured"; + } + + if (config.failure) { + decision.use_fallback = true; + decision.reason = config.failure_reason; + } + + std::error_code ec; + fs::path sysroot = fs::path(config.em_cache) / "sysroot"; + fs::path sysroot_stamp = fs::path(config.em_cache) / "sysroot_install.stamp"; + if (!fs::exists(sysroot, ec) || ec || !fs::exists(sysroot_stamp, ec) || ec) { + decision.use_fallback = true; + decision.reason = + "Emscripten sysroot not installed in cache: " + sysroot.string(); + } + + parse_warning_flags(user_args); + + bool compile_only = false; + bool is_wasm64 = false; + + if (!decision.use_fallback) { + if (has_header_inputs(user_args)) { + compile_only = true; + } + // Response files (@file) require complex tokenization (handling shell quoting, + // escaping, character encodings like UTF-8 with BOM, and recursive response + // file expansion). In LLVM/Clang, this is handled by llvm::cl::ExpandResponseFiles + // and llvm::cl::TokenizeGNUCommandLine / TokenizeWindowsCommandLine. Because + // emcc_native is a standalone executable without LLVM library dependencies, + // we fall back to Python (which uses shlex.split() in response_file.py) + // rather than maintaining a custom cross-platform tokenizer and encoding parser. + // TODO: Implement native response file expansion if we add LLVM dependencies + // or a robust lightweight tokenizer. + for (const auto& arg : user_args) { + if (arg.starts_with("@")) { + decision.use_fallback = true; + decision.reason = + "Response files (@file) not yet supported by native launcher"; + break; + } + if (arg == "-c" || arg == "-S" || arg == "-E" || arg == "-M" || + arg == "-MM" || arg == "-fsyntax-only") { + compile_only = true; + } else if (arg == "-m64" || arg == "-sMEMORY64" || + arg == "-sMEMORY64=1" || arg == "-sMEMORY64=2") { + is_wasm64 = true; + } + } + + // Pure compile step requires -c, -S, -E, -M, -MM, -fsyntax-only, or header compilation + if (!decision.use_fallback && !compile_only) { + decision.use_fallback = true; + decision.reason = + "No compile-only flag (-c, -S, -E) or header input found; defaulting to link phase"; + } + } + + std::vector filtered_user_args; + + if (!decision.use_fallback) { + for (size_t i = 0; i < user_args.size(); ++i) { + string_view arg = user_args[i]; + + string_view arg_base = arg; + size_t eq_pos = arg_base.find('='); + if (eq_pos != string_view::npos) { + arg_base = arg_base.substr(0, eq_pos); + } + + if (COMPLEX_OR_SYSTEM_FLAGS.contains(arg) || + COMPLEX_OR_SYSTEM_FLAGS.contains(arg_base)) { + decision.use_fallback = true; + decision.reason = "Contains Emscripten system or complex flag: " + std::string(arg); + break; + } + + if (arg == "-pthreads") { + decision.use_fallback = true; + decision.reason = "Invalid option -pthreads"; + break; + } + + if (is_emscripten_only_warning(arg)) { + continue; + } + + string_view setting_key; + bool ate_next = false; + if (is_dash_s_setting(user_args, i, setting_key, ate_next)) { + if (setting_key == "STRICT") { + if (ate_next) { + ++i; + } + continue; + } + if (COMPILE_TIME_SETTINGS.contains(setting_key)) { + decision.use_fallback = true; + decision.reason = + "Contains Emscripten compile-time setting: -s" + std::string(setting_key); + break; + } else { + // Linker-only setting: warn and ignore during compilation + emit_unused_warning("linker setting ignored during compilation: '" + + std::string(setting_key) + "'"); + if (ate_next) { + ++i; + } + continue; + } + } + + // Check for .bc output file suffix without -flto or -emit-llvm + if (arg == "-o" && i + 1 < user_args.size()) { + string_view out_path = user_args[i + 1]; + if (get_extension(out_path) == ".bc") { + bool has_lto_or_emit_llvm = false; + for (const auto& a : user_args) { + string_view sv_a(a); + if (sv_a.starts_with("-flto") || sv_a == "-emit-llvm") { + has_lto_or_emit_llvm = true; + break; + } + } + if (!has_lto_or_emit_llvm) { + decision.use_fallback = true; + decision.reason = + ".bc output file suffix used without -flto or -emit-llvm"; + break; + } + } + } + + if (arg == "-g4") { + decision.use_fallback = true; + decision.reason = "Contains deprecated debug flag: -g4"; + break; + } + + // Check if arg is a debug flag that Clang doesn't accept directly + if (arg == "-g1" || arg == "-g2") { + filtered_user_args.push_back("-g0"); + continue; + } + if (arg == "-gsource-map" || arg == "-gsource-map=inline" || + arg.starts_with("-gseparate-dwarf")) { + filtered_user_args.push_back("-g"); + continue; + } + + // Check if arg is a link-only flag (e.g. --js-library or + // --js-library=lib.js) + string_view flag_name = arg; + size_t eq = flag_name.find('='); + bool has_eq = (eq != string_view::npos); + if (has_eq) { + flag_name = flag_name.substr(0, eq); + } + + if (LINK_ONLY_FLAGS.contains(flag_name)) { + emit_unused_warning("linker flag ignored during compilation: '" + + std::string(arg) + "'"); + if (!has_eq && LINK_ONLY_FLAGS_WITH_ARGS.contains(flag_name) && + i + 1 < user_args.size() && !user_args[i + 1].starts_with("-")) { + ++i; + } + continue; + } + + filtered_user_args.push_back(arg); + } + } + + // Construct command vectors + if (decision.use_fallback) { + create_compiler_fallback_command(is_cxx, emscripten_root, user_args, decision); + } else { + bool is_asm_only = is_assembly_only(user_args); + create_clang_command( + is_cxx, is_wasm64, is_asm_only, filtered_user_args, config, decision); + + // Fall back if total command line length exceeds platform limits + size_t total_cmd_len = decision.target_binary.size(); + for (const auto& a : decision.target_args) { + total_cmd_len += a.size() + 1; + } +#ifdef _WIN32 + constexpr size_t MAX_CMD_LEN = 8192; +#else + constexpr size_t MAX_CMD_LEN = 32768; +#endif + if (total_cmd_len > MAX_CMD_LEN) { + decision.target_args.clear(); + decision.use_fallback = true; + decision.reason = "Command line length (" + std::to_string(total_cmd_len) + + " chars) exceeds limit (" + std::to_string(MAX_CMD_LEN) + + "); falling back to Python driver for response file handling"; + create_compiler_fallback_command(is_cxx, emscripten_root, user_args, decision); + } + } + + return decision; +} + +void apply_ccache_wrapper(DriverDecision& decision) { + const char* emcc_ccache = std::getenv("_EMCC_CCACHE"); + if (!emcc_ccache || emcc_ccache[0] == '\0') { + return; + } + unsetenv("_EMCC_CCACHE"); + + decision.target_args.insert(decision.target_args.begin(), decision.target_binary); +#ifdef _WIN32 + decision.target_binary = "ccache.exe"; +#else + decision.target_binary = "ccache"; +#endif +} + +} // namespace emscripten diff --git a/tools/emcc_native/driver.h b/tools/emcc_native/driver.h new file mode 100644 index 0000000000000..49a02b676a9d8 --- /dev/null +++ b/tools/emcc_native/driver.h @@ -0,0 +1,38 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_DRIVER_H +#define EMCC_NATIVE_DRIVER_H + +#include "config.h" +#include +#include + +namespace emscripten { + +struct DriverDecision { + bool use_fallback = false; + std::string target_binary; + std::vector target_args; + std::string reason; +}; + +// Get the Python executable path (from EMSDK_PYTHON or default). +std::string get_python_executable(); + +DriverDecision analyze_request(std::string_view driver_arg0, + const fs::path& emscripten_root, + const fs::path& exe_path, + const std::vector& user_args, + const Config& config); + +// If _EMCC_CCACHE is set, unsets it and transforms target_binary to ccache. +void apply_ccache_wrapper(DriverDecision& decision); + +} // namespace emscripten + +#endif // EMCC_NATIVE_DRIVER_H diff --git a/tools/emcc_native/exec.cpp b/tools/emcc_native/exec.cpp new file mode 100644 index 0000000000000..d6e54efe4bdd1 --- /dev/null +++ b/tools/emcc_native/exec.cpp @@ -0,0 +1,118 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "exec.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace emscripten { + +// Quotes a command line argument for Windows CreateProcess / CommandLineToArgvW. +// +// Arguments that are empty or contain spaces, tabs, or double quotes must be +// wrapped in double quotes. According to standard Windows command-line parsing +// rules (CommandLineToArgvW): +// - 2N backslashes followed by a double quote produce N literal backslashes and +// a string quote delimiter (start/end of quote). +// - (2N + 1) backslashes followed by a double quote produce N literal +// backslashes and a literal double quote character ("). +// - Backslashes not followed by a double quote are literal and are not doubled. +std::string quote_for_windows(string_view arg) { + if (!arg.empty() && arg.find_first_of(" \t\"") == string_view::npos) { + return std::string(arg); + } + std::string quoted = "\""; + for (size_t i = 0; i < arg.size(); ++i) { + size_t num_backslashes = 0; + while (i < arg.size() && arg[i] == '\\') { + num_backslashes++; + i++; + } + if (i == arg.size()) { + quoted.append(num_backslashes * 2, '\\'); + break; + } + if (arg[i] == '\"') { + quoted.append(num_backslashes * 2 + 1, '\\'); + quoted.push_back('\"'); + } else { + quoted.append(num_backslashes, '\\'); + quoted.push_back(arg[i]); + } + } + quoted += "\""; + return quoted; +} + +[[noreturn]] void exec_process(const std::string& binary, + const std::vector& args) { + unsetenv("_PYTHON_SYSCONFIGDATA_NAME"); + +#ifdef _WIN32 + if (GetEnvironmentVariableW(L"EM_WORKAROUND_PYTHON_BUG_34780", nullptr, 0) > 0) { + CloseHandle(GetStdHandle(STD_INPUT_HANDLE)); + } + + std::string cmdline = quote_for_windows(binary); + for (const auto& arg : args) { + cmdline += " " + quote_for_windows(arg); + } + + int wlen = MultiByteToWideChar(CP_UTF8, 0, cmdline.c_str(), -1, nullptr, 0); + if (wlen == 0) { + std::cerr << "emcc_native: error converting command line to UTF-16" << std::endl; + std::exit(1); + } + std::vector wcmdline(wlen); + MultiByteToWideChar(CP_UTF8, 0, cmdline.c_str(), -1, wcmdline.data(), wlen); + + STARTUPINFOW si; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + PROCESS_INFORMATION pi; + ZeroMemory(&pi, sizeof(pi)); + + if (!CreateProcessW(nullptr, wcmdline.data(), nullptr, nullptr, TRUE, 0, nullptr, nullptr, &si, &pi)) { + DWORD err = GetLastError(); + std::cerr << "emcc_native: error executing " << binary + << " (CreateProcessW failed: " << err << ")" << std::endl; + std::exit(1); + } + + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code = 0; + GetExitCodeProcess(pi.hProcess, &exit_code); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + std::exit(static_cast(exit_code)); +#else + std::vector argv; + argv.reserve(args.size() + 2); + argv.push_back(binary.c_str()); + for (const auto& arg : args) { + argv.push_back(arg.c_str()); + } + argv.push_back(nullptr); + + execvp(binary.c_str(), const_cast(argv.data())); + + std::cerr << "emcc_native: error executing " << binary << ": " + << std::strerror(errno) << std::endl; + std::exit(1); +#endif +} + +} // namespace emscripten diff --git a/tools/emcc_native/exec.h b/tools/emcc_native/exec.h new file mode 100644 index 0000000000000..cd08a6cc76dfb --- /dev/null +++ b/tools/emcc_native/exec.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_EXEC_H +#define EMCC_NATIVE_EXEC_H + +#include +#include +#include +#include + +#ifdef _WIN32 +inline int unsetenv(const char* name) { + return _putenv_s(name, ""); +} +#endif + +namespace emscripten { + +using std::string_view; + +// Quote a command line argument for Windows _spawnvp / CreateProcess. +std::string quote_for_windows(string_view arg); + +// Execute the specified binary with args, replacing the current process or +// exiting with the child's return code. Does not return. +[[noreturn]] void exec_process(const std::string& binary, + const std::vector& args); + +} // namespace emscripten + +#endif // EMCC_NATIVE_EXEC_H diff --git a/tools/emcc_native/gen_settings.py b/tools/emcc_native/gen_settings.py new file mode 100755 index 0000000000000..d3fdfde361a20 --- /dev/null +++ b/tools/emcc_native/gen_settings.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +"""Generate C++ header tools/emcc_native/generated_settings.h from Python flag definitions.""" + +import os +import sys + +script_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(os.path.dirname(script_dir)) +sys.path.insert(0, root_dir) + +from emcc import LINK_ONLY_FLAGS +from tools import diagnostics +from tools.settings import COMPILE_TIME_SETTINGS +from tools.utils import path_from_root, read_file, write_file + +HEADER_PATH = path_from_root('tools/emcc_native/generated_settings.h') + + +def generate(check_only=False): + link_flags = sorted(LINK_ONLY_FLAGS) + compile_settings = sorted(COMPILE_TIME_SETTINGS) + ems_warnings = sorted(name for name, info in diagnostics.manager.warnings.items() if not info['shared']) + + lines = [ + '/*', + ' * Copyright 2026 The Emscripten Authors. All rights reserved.', + ' * Emscripten is available under two separate licenses, the MIT license and the', + ' * University of Illinois/NCSA Open Source License. Both these licenses can be', + ' * found in the LICENSE file.', + ' *', + ' * Auto-generated by tools/emcc_native/gen_settings.py. DO NOT EDIT.', + ' */', + '', + '#ifndef EMCC_NATIVE_GENERATED_SETTINGS_H', + '#define EMCC_NATIVE_GENERATED_SETTINGS_H', + '', + '#include ', + '#include ', + '', + 'namespace emscripten {', + '', + 'inline const std::unordered_set LINK_ONLY_FLAGS = {', + ] + + for flag in link_flags: + lines.append(f' "{flag}",') + lines.extend([ + '};', + '', + 'inline const std::unordered_set COMPILE_TIME_SETTINGS = {', + ]) + + for setting in compile_settings: + lines.append(f' "{setting}",') + lines.extend([ + '};', + '', + 'inline const std::unordered_set EMSCRIPTEN_ONLY_WARNINGS = {', + ]) + + for warning in ems_warnings: + lines.append(f' "{warning}",') + lines.extend([ + '};', + '', + '} // namespace emscripten', + '', + '#endif // EMCC_NATIVE_GENERATED_SETTINGS_H', + ]) + + content = '\n'.join(lines) + '\n' + + if check_only: + existing = read_file(HEADER_PATH) + if existing != content: + print(f'Error: {HEADER_PATH} is out of date.', file=sys.stderr) + print('Run tools/emcc_native/gen_settings.py to update it.', file=sys.stderr) + sys.exit(1) + print(f'{HEADER_PATH} is up to date.') + else: + write_file(HEADER_PATH, content) + print(f'Wrote {HEADER_PATH}') + + +if __name__ == '__main__': + check = '--check' in sys.argv + generate(check_only=check) diff --git a/tools/emcc_native/generated_settings.h b/tools/emcc_native/generated_settings.h new file mode 100644 index 0000000000000..abead2d118ad3 --- /dev/null +++ b/tools/emcc_native/generated_settings.h @@ -0,0 +1,117 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + * + * Auto-generated by tools/emcc_native/gen_settings.py. DO NOT EDIT. + */ + +#ifndef EMCC_NATIVE_GENERATED_SETTINGS_H +#define EMCC_NATIVE_GENERATED_SETTINGS_H + +#include +#include + +namespace emscripten { + +inline const std::unordered_set LINK_ONLY_FLAGS = { + "--bind", + "--closure", + "--cpuprofiler", + "--embed-file", + "--emit-symbol-map", + "--emrun", + "--exclude-file", + "--extern-post-js", + "--extern-pre-js", + "--ignore-dynamic-linking", + "--js-library", + "--js-transform", + "--oformat", + "--output-eol", + "--output_eol", + "--post-js", + "--pre-js", + "--preload-file", + "--profiling-funcs", + "--proxy-to-worker", + "--shell-file", + "--source-map-base", + "--threadprofiler", + "--use-preload-plugins", +}; + +inline const std::unordered_set COMPILE_TIME_SETTINGS = { + "DEBUG_LEVEL", + "DISABLE_EXCEPTION_CATCHING", + "DISABLE_EXCEPTION_THROWING", + "EMSCRIPTEN_TRACING", + "EXCEPTION_CATCHING_ALLOWED", + "INLINING_LIMIT", + "LINKABLE", + "LTO", + "MAIN_MODULE", + "MEMORY64", + "OPT_LEVEL", + "PTHREADS", + "SDL2_IMAGE_FORMATS", + "SDL2_MIXER_FORMATS", + "SHARED_MEMORY", + "SIDE_MODULE", + "STRICT", + "SUPPORT_LONGJMP", + "USE_BOOST_HEADERS", + "USE_BULLET", + "USE_BZIP2", + "USE_COCOS2D", + "USE_FREETYPE", + "USE_GIFLIB", + "USE_HARFBUZZ", + "USE_ICU", + "USE_LIBJPEG", + "USE_LIBPNG", + "USE_MODPLUG", + "USE_MPG123", + "USE_OGG", + "USE_PTHREADS", + "USE_REGAL", + "USE_SDL", + "USE_SDL_GFX", + "USE_SDL_IMAGE", + "USE_SDL_MIXER", + "USE_SDL_NET", + "USE_SDL_TTF", + "USE_SQLITE3", + "USE_VORBIS", + "USE_ZLIB", + "WASM_EXCEPTIONS", + "WASM_LEGACY_EXCEPTIONS", + "WASM_OBJECT_FILES", + "WASM_WORKERS", +}; + +inline const std::unordered_set EMSCRIPTEN_ONLY_WARNINGS = { + "absolute-paths", + "almost-asm", + "closure", + "compatibility", + "em-js-i64", + "emcc", + "experimental", + "export-main", + "js-compiler", + "legacy-settings", + "limited-postlink-optimizations", + "linkflags", + "map-unrecognized-libraries", + "pthreads-mem-growth", + "undefined", + "unsupported", + "unused-main", + "version-check", +}; + +} // namespace emscripten + +#endif // EMCC_NATIVE_GENERATED_SETTINGS_H diff --git a/tools/emcc_native/main.cpp b/tools/emcc_native/main.cpp new file mode 100644 index 0000000000000..c8467d7d9bffe --- /dev/null +++ b/tools/emcc_native/main.cpp @@ -0,0 +1,146 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "config.h" +#include "driver.h" +#include "exec.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#elif defined(__APPLE__) +#include +#include +#else +#include +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define UNREACHABLE() __assume(0) +#else +#define UNREACHABLE() __builtin_unreachable() +#endif + +namespace fs = std::filesystem; + +using namespace emscripten; + +namespace { + +template void errlog(Args&&... args) { + (std::cerr << ... << std::forward(args)) << std::endl; +} + + +fs::path get_self_executable_path() { +#if defined(_WIN32) + std::vector buf(MAX_PATH); + DWORD len = GetModuleFileNameW(NULL, buf.data(), static_cast(buf.size())); + while (len >= buf.size()) { + buf.resize(buf.size() * 2); + len = GetModuleFileNameW(NULL, buf.data(), static_cast(buf.size())); + } + if (len == 0) { + errlog("emcc_native: error: GetModuleFileNameW failed"); + std::exit(1); + } + return fs::path(buf.data()); +#elif defined(__linux__) + std::error_code ec; + fs::path proc_path = fs::read_symlink("/proc/self/exe", ec); + if (ec || proc_path.empty()) { + errlog("emcc_native: error: reading /proc/self/exe failed: ", ec.message()); + std::exit(1); + } + return proc_path; +#elif defined(__APPLE__) + uint32_t size = 1024; + std::vector buf(size); + if (_NSGetExecutablePath(buf.data(), &size) != 0) { + buf.resize(size); + if (_NSGetExecutablePath(buf.data(), &size) != 0) { + errlog("emcc_native: error: _NSGetExecutablePath failed"); + std::exit(1); + } + } + return fs::path(buf.data()); +#else +#error "Unsupported platform for get_self_executable_path" +#endif +} + +fs::path find_emscripten_root(const fs::path& exe_path) { + fs::path dir = fs::weakly_canonical(exe_path).parent_path(); + while (!dir.empty() && dir != dir.root_path()) { + if (fs::exists(dir / "emcc.py")) { + return dir; + } + dir = dir.parent_path(); + } + + errlog("emcc_native: error: could not locate Emscripten root directory " + "(emcc.py not found relative to launcher binary at ", + exe_path.generic_string(), + ")"); + std::exit(1); +} + +void log_decision(const DriverDecision& decision) { + if (decision.use_fallback) { + errlog("emcc_native: falling back to python driver (", decision.reason, ")"); + } else { + errlog("emcc_native: executing clang directly"); + } + + std::string full_cmd = decision.target_binary; + for (const auto& arg : decision.target_args) { + full_cmd += " " + quote_for_windows(arg); + } + + errlog("emcc_native: exec: ", full_cmd); +} + +} // namespace + +int main(int argc, char** argv) { + assert(argc >= 1); + + fs::path exe_path = get_self_executable_path(); + fs::path emscripten_root = find_emscripten_root(exe_path); + Config config = load_config(emscripten_root); + + std::vector user_args(argv + 1, argv + argc); + + auto decision = analyze_request(argv[0], emscripten_root, exe_path, user_args, config); + + const char* native_env = std::getenv("EMCC_NATIVE"); + if (decision.use_fallback && native_env && std::string(native_env) == "1") { + errlog("emcc_native: error: falling back to python driver with EMCC_NATIVE=1 (", + decision.reason, + ")"); + return 1; + } + + apply_ccache_wrapper(decision); + + const char* emcc_debug = std::getenv("EMCC_DEBUG"); + const char* native_debug = std::getenv("EMCC_NATIVE_DEBUG"); + if ((emcc_debug && emcc_debug[0] != '\0') || (native_debug && native_debug[0] != '\0')) { + log_decision(decision); + } + + exec_process(decision.target_binary, decision.target_args); + UNREACHABLE(); +} diff --git a/tools/emcc_native/tests/test_native.cpp b/tools/emcc_native/tests/test_native.cpp new file mode 100644 index 0000000000000..ee74e32f0e510 --- /dev/null +++ b/tools/emcc_native/tests/test_native.cpp @@ -0,0 +1,673 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "config.h" +#include "driver.h" +#include "exec.h" + +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace emscripten; + +namespace { + +void set_env(const std::string& name, const std::string& value) { +#ifdef _WIN32 + _putenv_s(name.c_str(), value.c_str()); +#else + setenv(name.c_str(), value.c_str(), 1); +#endif +} + +void unset_env(const std::string& name) { +#ifdef _WIN32 + _putenv_s(name.c_str(), ""); +#else + unsetenv(name.c_str()); +#endif +} + +DriverDecision analyze_compiler(std::string_view tool, + const std::vector& user_args, + const Config& config, + const fs::path& emscripten_root = "/emsdk/emscripten") { + return analyze_request(tool, emscripten_root, emscripten_root / tool, user_args, config); +} + +} // namespace + +void test_config_parsing() { + fs::path temp_dir = fs::temp_directory_path() / "emcc_native_test"; + fs::create_directories(temp_dir); + + fs::path config_file = temp_dir / ".emscripten"; + { + std::ofstream out(config_file); + out << "# Test config\n"; + out << "LLVM_ROOT = '/custom/llvm/bin' # trailing comment\n"; + out << "CACHE = '/custom/cache'\n"; + } + + set_env("EMSDK_PYTHON", "python3.11"); + + assert(load_config(temp_dir).llvm_root == "/custom/llvm/bin"); + assert(load_config(temp_dir).em_cache == "/custom/cache"); + assert(get_python_executable() == "python3.11"); + + // Test two-levels-up embedded config file lookup + fs::path emsdk_root = temp_dir / "emsdk"; + fs::path nested_dir = emsdk_root / "upstream" / "emscripten"; + fs::create_directories(nested_dir); + fs::path embedded_config = emsdk_root / ".emscripten"; + { + std::ofstream out(embedded_config); + out << "LLVM_ROOT = '/embedded/llvm/bin'\n"; + } + assert(find_config_file(nested_dir) == embedded_config); + assert(load_config(nested_dir).llvm_root == "/embedded/llvm/bin"); + + // Test environment variable expansion (including $CFGDIR, $VAR, ${VAR}) + fs::path var_dir = temp_dir / "var_test"; + fs::create_directories(var_dir); + fs::path var_config = var_dir / ".emscripten"; + set_env("TEST_LLVM_DIR", "/env_llvm"); + { + std::ofstream out(var_config); + out << "LLVM_ROOT = '$CFGDIR/bin'\n"; + out << "CACHE = '${TEST_LLVM_DIR}/cache'\n"; + } + assert(load_config(var_dir).llvm_root == (var_dir / "bin").string()); + assert(load_config(var_dir).em_cache == "/env_llvm/cache"); + + // Test Python expression evaluation (emsdk_path, os.path.dirname, os.path.abspath, +) + fs::path py_dir = temp_dir / "py_test"; + fs::create_directories(py_dir); + fs::path py_config = py_dir / ".emscripten"; + { + std::ofstream out(py_config); + out << "import os\n"; + out << "emsdk_path = os.path.dirname(os.path.abspath(__file__))\n"; + out << "LLVM_ROOT = emsdk_path + '/upstream/bin'\n"; + out << "CACHE = emsdk_path + '/cache'\n"; + } + assert(load_config(py_dir).failure == true); + assert(load_config(py_dir).failure_reason.find("Complex expression in config file") != std::string::npos); + + unset_env("EMSDK_PYTHON"); + + fs::remove_all(temp_dir); + std::cout << "[PASS] test_config_parsing" << std::endl; +} + +std::string get_test_cache() { + fs::path p = fs::temp_directory_path() / "emcc_native_test_cache"; + fs::create_directories(p / "sysroot"); + std::ofstream stamp(p / "sysroot_install.stamp"); + return p.string(); +} + +void test_driver_decision_compile_only() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "hello.c", "-o", "hello.o"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + assert(dec.target_args[0] == "-target"); + assert(dec.target_args[1] == "wasm32-unknown-emscripten"); + assert(dec.target_args[2] == "-fignore-exceptions"); + assert(dec.target_args[9] == "--sysroot=" + cfg.em_cache + "/sysroot"); + assert(dec.target_args[14] == "-c"); + assert(dec.target_args[15] == "hello.c"); + assert(dec.target_args[16] == "-o"); + assert(dec.target_args[17] == "hello.o"); + + std::cout << "[PASS] test_driver_decision_compile_only" << std::endl; +} + +void test_driver_decision_cxx_and_wasm64() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "hello.cpp", "-o", "hello.o", "-m64"}; + DriverDecision dec = analyze_compiler("em++", args, cfg); + + assert(!dec.use_fallback); + assert(dec.target_args[0] == "-target"); + assert(dec.target_args[1] == "wasm64-unknown-emscripten"); + + std::cout << "[PASS] test_driver_decision_cxx_and_wasm64" << std::endl; +} + +void test_driver_decision_header_and_syntax_only() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args1 = {"-xc++-header", "header.h", "-o", "header.h.gch"}; + DriverDecision dec1 = analyze_compiler("em++", args1, cfg); + assert(!dec1.use_fallback); + + std::vector args2 = {"header.hpp", "-o", "header.hpp.pch"}; + DriverDecision dec2 = analyze_compiler("em++", args2, cfg); + assert(!dec2.use_fallback); + + std::vector args3 = {"-fsyntax-only", "test.cpp"}; + DriverDecision dec3 = analyze_compiler("em++", args3, cfg); + assert(!dec3.use_fallback); + + std::cout << "[PASS] test_driver_decision_header_and_syntax_only" << std::endl; +} + +void test_driver_decision_fallback_link() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"hello.o", "-o", "hello.js"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(dec.use_fallback); +#ifdef _WIN32 + assert(dec.target_binary == "python.exe"); + assert(dec.target_args[0] == "-E"); + assert(dec.target_args[1] == "-X"); + assert(dec.target_args[2] == "utf8"); + assert(dec.target_args[3] == "/emsdk/emscripten/emcc.py"); +#else + assert(dec.target_binary == "python3"); + assert(dec.target_args[0] == "-E"); + assert(dec.target_args[1] == "/emsdk/emscripten/emcc.py"); +#endif + + std::cout << "[PASS] test_driver_decision_fallback_link" << std::endl; +} + +void test_driver_decision_ignore_linker_flags_and_settings_during_compile() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", + "hello.c", + "-o", + "hello.o", + "--js-library", + "lib.js", + "--embed-file=file", + "-sEXPORTED_FUNCTIONS=['_main']"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + assert(dec.target_args[0] == "-target"); + assert(dec.target_args[1] == "wasm32-unknown-emscripten"); + assert(dec.target_args[14] == "-c"); + assert(dec.target_args[15] == "hello.c"); + assert(dec.target_args[16] == "-o"); + assert(dec.target_args[17] == "hello.o"); + + std::cout + << "[PASS] " + "test_driver_decision_ignore_linker_flags_and_settings_during_compile" + << std::endl; +} + +void test_driver_decision_bind_flag() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "foo.cpp", "--bind", "bar.cpp"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + bool has_foo = false, has_bar = false, has_bind = false; + for (const auto& arg : dec.target_args) { + if (arg == "foo.cpp") has_foo = true; + if (arg == "bar.cpp") has_bar = true; + if (arg == "--bind") has_bind = true; + } + assert(has_foo); + assert(has_bar); + assert(!has_bind); + + std::cout << "[PASS] test_driver_decision_bind_flag" << std::endl; +} + +void test_driver_decision_linker_dash_s_setting() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + // Test space-separated setting value: -s EXPORTED_FUNCTIONS=['_main'] + std::vector args = {"-c", "foo.cpp", "-s", "EXPORTED_FUNCTIONS=['_main']", "bar.cpp"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + bool has_foo = false, has_bar = false, has_val = false; + for (const auto& arg : dec.target_args) { + if (arg == "foo.cpp") has_foo = true; + if (arg == "bar.cpp") has_bar = true; + if (arg == "EXPORTED_FUNCTIONS=['_main']") has_val = true; + } + assert(has_foo); + assert(has_bar); + assert(!has_val); + + // Test joined setting value: -sEXPORTED_FUNCTIONS=['_main'] + args = {"-c", "foo.cpp", "-sEXPORTED_FUNCTIONS=['_main']", "bar.cpp"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + has_foo = false; has_bar = false; has_val = false; + for (const auto& arg : dec.target_args) { + if (arg == "foo.cpp") has_foo = true; + if (arg == "bar.cpp") has_bar = true; + if (arg == "-sEXPORTED_FUNCTIONS=['_main']") has_val = true; + } + assert(has_foo); + assert(has_bar); + assert(!has_val); + + std::cout << "[PASS] test_driver_decision_linker_dash_s_setting" << std::endl; +} + +void test_driver_decision_fallback_compile_time_setting() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = { + "-c", "hello.c", "-o", "hello.o", "-sDISABLE_EXCEPTION_CATCHING=0"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + + // Settings containing numbers (e.g. MEMORY64, SDL2_IMAGE_FORMATS) must be recognized + args = {"-c", "hello.c", "-o", "hello.o", "-sMEMORY64=1"}; + dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.o", "-sSDL2_IMAGE_FORMATS=['png']"}; + dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.o", "--em-config", "/custom/.emscripten"}; + dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.o", "--em-config=/custom/.emscripten"}; + dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + + std::cout << "[PASS] test_driver_decision_fallback_compile_time_setting" + << std::endl; +} + +void test_driver_decision_wno_unused_command_line_argument() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", + "hello.c", + "-o", + "hello.o", + "--js-library", + "lib.js", + "-Wno-unused-command-line-argument"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + assert(dec.target_args[14] == "-c"); + assert(dec.target_args[15] == "hello.c"); + assert(dec.target_args[16] == "-o"); + assert(dec.target_args[17] == "hello.o"); + assert(dec.target_args[18] == "-Wno-unused-command-line-argument"); + + std::cout << "[PASS] test_driver_decision_wno_unused_command_line_argument" + << std::endl; +} + +void test_driver_decision_emcc_native_override() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + set_env("EMCC_NATIVE", "0"); + std::vector args = {"-c", "hello.c", "-o", "hello.o"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason == "EMCC_NATIVE set to disable native launcher"); + + set_env("EMCC_NATIVE", "1"); + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + unset_env("EMCC_NATIVE"); + + std::cout << "[PASS] test_driver_decision_emcc_native_override" << std::endl; +} + +void test_driver_decision_simd_flags() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = { + "-c", "hello.c", "-o", "hello.o", "-msimd128", "-msse2"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + bool has_sse = false, has_sse2 = false; + for (const auto& arg : dec.target_args) { + if (arg == "-D__SSE__=1") + has_sse = true; + if (arg == "-D__SSE2__=1") + has_sse2 = true; + } + assert(has_sse); + assert(has_sse2); + + std::cout << "[PASS] test_driver_decision_simd_flags" << std::endl; +} + +void test_driver_decision_pthread() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = { + "-c", "hello.c", "-o", "hello.o", "-pthread"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + bool has_pthread = false; + for (const auto& arg : dec.target_args) { + if (arg == "-pthread") + has_pthread = true; + } + assert(has_pthread); + + std::cout << "[PASS] test_driver_decision_pthread" << std::endl; +} + +void test_driver_decision_emscripten_only_warning_flags() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = { + "-c", "hello.c", "-o", "hello.o", "-Wclosure", "-Wno-limited-postlink-optimizations", "-Werror"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + + assert(!dec.use_fallback); + bool has_closure = false, has_limited = false, has_error = false; + for (const auto& arg : dec.target_args) { + if (arg == "-Wclosure") has_closure = true; + if (arg == "-Wno-limited-postlink-optimizations") has_limited = true; + if (arg == "-Werror") has_error = true; + } + assert(!has_closure); + assert(!has_limited); + assert(has_error); + + std::cout << "[PASS] test_driver_decision_emscripten_only_warning_flags" << std::endl; +} + +void test_driver_decision_cmdline_length_fallback() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "hello.c", "-o", "hello.o"}; + std::string long_flag = "-I/path/to/very/long/include/directory/"; + for (int i = 0; i < 1500; ++i) { + args.push_back(long_flag + std::to_string(i)); + } + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason.find("Command line length") != std::string::npos); + + std::cout << "[PASS] test_driver_decision_cmdline_length_fallback" << std::endl; +} + +void test_driver_decision_strict_setting() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "hello.c", "-o", "hello.o", "-sSTRICT"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.o", "-sSTRICT=1"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.o", "-s", "STRICT"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + std::cout << "[PASS] test_driver_decision_strict_setting" << std::endl; +} + +void test_driver_decision_lto() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "hello.c", "-o", "hello.o", "-flto"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + bool has_flto = false; + for (const auto& arg : dec.target_args) { + if (arg == "-flto") has_flto = true; + } + assert(has_flto); + + args = {"-c", "hello.c", "-o", "hello.o", "-flto=thin"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.o", "-fno-lto"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.bc"}; + dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason.find( + ".bc output file suffix used without -flto or -emit-llvm") != + std::string::npos); + + args = {"-c", "hello.c", "-o", "hello.bc", "-flto"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + args = {"-c", "hello.c", "-o", "hello.bc", "-emit-llvm"}; + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + std::cout << "[PASS] test_driver_decision_lto" << std::endl; +} + +void test_driver_decision_response_files() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + std::vector args = {"-c", "hello.c", "-o", "hello.o", "@args.rsp"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason == "Response files (@file) not yet supported by native launcher"); + + std::cout << "[PASS] test_driver_decision_response_files" << std::endl; +} + +void test_driver_decision_config_failure() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + cfg.failure = true; + cfg.failure_reason = + "Complex expression in config file for LLVM_ROOT: emsdk_path + '/upstream/bin'"; + + std::vector args = {"-c", "hello.c"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason.find("Complex expression in config file") != std::string::npos); + + std::cout << "[PASS] test_driver_decision_config_failure" << std::endl; +} + +void test_quote_for_windows() { + assert(quote_for_windows("hello") == "hello"); + assert(quote_for_windows("hello world") == "\"hello world\""); + assert(quote_for_windows("foo\\bar") == "foo\\bar"); + assert(quote_for_windows("foo\\bar\\ baz") == "\"foo\\bar\\ baz\""); + assert(quote_for_windows("foo\\bar\\") == "foo\\bar\\"); + assert(quote_for_windows("foo bar\\") == "\"foo bar\\\\\""); + assert(quote_for_windows("foo \"bar\"") == "\"foo \\\"bar\\\"\""); + assert(quote_for_windows("foo \\\"bar\"") == "\"foo \\\\\\\"bar\\\"\""); + assert(quote_for_windows("") == "\"\""); + std::cout << "[PASS] test_quote_for_windows" << std::endl; +} + +void test_driver_decision_missing_sysroot() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + fs::path p = fs::temp_directory_path() / "emcc_native_test_cache_missing"; + fs::remove_all(p); + fs::create_directories(p); + cfg.em_cache = p.string(); + + std::vector args = {"-c", "hello.c", "-o", "hello.o"}; + DriverDecision dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason.find("Emscripten sysroot not installed in cache") != std::string::npos); + + fs::create_directories(p / "sysroot"); + dec = analyze_compiler("emcc", args, cfg); + assert(dec.use_fallback); + assert(dec.reason.find("Emscripten sysroot not installed in cache") != std::string::npos); + + std::ofstream stamp(p / "sysroot_install.stamp"); + dec = analyze_compiler("emcc", args, cfg); + assert(!dec.use_fallback); + + fs::remove_all(p); + std::cout << "[PASS] test_driver_decision_missing_sysroot" << std::endl; +} + +void test_apply_ccache_wrapper() { + DriverDecision dec; + dec.target_binary = "/path/to/clang"; + dec.target_args = {"-c", "hello.c"}; + + set_env("_EMCC_CCACHE", "1"); + apply_ccache_wrapper(dec); + + assert(std::getenv("_EMCC_CCACHE") == nullptr); +#ifdef _WIN32 + assert(dec.target_binary == "ccache.exe"); +#else + assert(dec.target_binary == "ccache"); +#endif + assert(dec.target_args.size() == 3); + assert(dec.target_args[0] == "/path/to/clang"); + assert(dec.target_args[1] == "-c"); + assert(dec.target_args[2] == "hello.c"); + + std::cout << "[PASS] test_apply_ccache_wrapper" << std::endl; +} + +void test_non_compiler_tool_launching() { + Config cfg; + cfg.llvm_root = "/emsdk/llvm/bin"; + cfg.em_cache = get_test_cache(); + + fs::path dummy_root = fs::temp_directory_path() / "emcc_native_test_root"; + fs::create_directories(dummy_root / "tools"); + fs::create_directories(dummy_root / "test"); + std::ofstream(dummy_root / "emar.py") << "# dummy"; + std::ofstream(dummy_root / "tools" / "emprofile.py") << "# dummy"; + std::ofstream(dummy_root / "test" / "runner.py") << "# dummy"; + + std::vector args1 = {"create", "libfoo.a", "bar.o"}; + DriverDecision dec1 = analyze_request("emar", dummy_root, dummy_root / "emar", args1, cfg); + assert(dec1.use_fallback); +#ifdef _WIN32 + assert(dec1.target_binary == "python.exe"); + assert(dec1.target_args[0] == "-E"); + assert(dec1.target_args[1] == "-X"); + assert(dec1.target_args[2] == "utf8"); + assert(dec1.target_args[3].find("emar.py") != std::string::npos); +#else + assert(dec1.target_binary == "python3"); + assert(dec1.target_args[0] == "-E"); + assert(dec1.target_args[1].find("emar.py") != std::string::npos); +#endif + +#ifdef _WIN32 + constexpr size_t script_idx = 3; +#else + constexpr size_t script_idx = 1; +#endif + + std::vector args2 = {"--help"}; + DriverDecision dec2 = analyze_request("emprofile", dummy_root, dummy_root / "emprofile", args2, cfg); + assert(dec2.use_fallback); + assert(dec2.target_args[script_idx].find("emprofile.py") != std::string::npos); + + std::vector args3 = {"--help"}; + DriverDecision dec3 = analyze_request("runner", dummy_root, dummy_root / "runner", args3, cfg); + assert(dec3.use_fallback); + assert(dec3.target_args[script_idx].find("runner.py") != std::string::npos); + + fs::remove_all(dummy_root); + std::cout << "[PASS] test_non_compiler_tool_launching" << std::endl; +} + +int main() { + std::cout << "Running emcc-native unit tests..." << std::endl; + test_config_parsing(); + test_quote_for_windows(); + test_driver_decision_compile_only(); + test_driver_decision_cxx_and_wasm64(); + test_driver_decision_header_and_syntax_only(); + test_driver_decision_fallback_link(); + test_driver_decision_ignore_linker_flags_and_settings_during_compile(); + test_driver_decision_bind_flag(); + test_driver_decision_linker_dash_s_setting(); + test_driver_decision_fallback_compile_time_setting(); + test_driver_decision_strict_setting(); + test_driver_decision_lto(); + test_driver_decision_wno_unused_command_line_argument(); + test_driver_decision_simd_flags(); + test_driver_decision_pthread(); + test_driver_decision_emscripten_only_warning_flags(); + test_driver_decision_cmdline_length_fallback(); + test_driver_decision_emcc_native_override(); + test_driver_decision_response_files(); + test_driver_decision_config_failure(); + test_driver_decision_missing_sysroot(); + test_apply_ccache_wrapper(); + test_non_compiler_tool_launching(); + std::filesystem::remove_all(fs::temp_directory_path() / "emcc_native_test_cache"); + std::cout << "All emcc-native tests passed successfully!" << std::endl; + return 0; +} diff --git a/tools/maint/create_entry_points.py b/tools/maint/create_entry_points.py index ab9b144f21cc7..ded8450d7b580 100755 --- a/tools/maint/create_entry_points.py +++ b/tools/maint/create_entry_points.py @@ -6,15 +6,15 @@ """Tool for creating/maintaining the python launcher scripts for emscripten tools. -This tool makes copies or `run_python.sh/.bat` and `run_python_compiler.sh/.bat` +Note: This tool is deprecated and being replaced by emcc_native (built via `bootstrap.py`). + +This tool makes copies of `run_python.sh/.bat` and `run_python_compiler.sh/.bat` script for each entry point. On UNIX we previously used symbolic links for simplicity but this breaks MINGW users on windows who want to use the shell script launcher but don't have symlink support. """ import os -import platform -import shutil import stat import sys @@ -61,18 +61,13 @@ } -windows_exe = os.path.join(__rootdir__, 'tools/pylauncher/pylauncher.exe') -if platform.machine().lower() in {'arm64', 'aarch64'}: - windows_exe = os.path.join(__rootdir__, 'tools/pylauncher/pylauncher-arm64.exe') - - def make_executable(filename): old_mode = stat.S_IMODE(os.stat(filename).st_mode) os.chmod(filename, old_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) def maybe_remove(filename): - if os.path.exists(filename): + if os.path.lexists(filename): os.remove(filename) @@ -82,13 +77,13 @@ def read_file(filename): def write_file(filename, content): + maybe_remove(filename) with open(filename, 'w', encoding='utf-8') as f: f.write(content) -def main(all_platforms, use_bat_file): - if 'EM_USE_BAT_FILES' in os.environ: - use_bat_file = True +def main(all_platforms): + print('warning: create_entry_points.py is deprecated and being replaced by emcc_native', file=sys.stderr) is_windows = sys.platform.startswith('win') is_msys2 = 'MSYSTEM' in os.environ do_unix = all_platforms or not is_windows or is_msys2 @@ -117,17 +112,12 @@ def generate_entry_points(cmd, path): make_executable(launcher) if do_windows: - maybe_remove(launcher + '.ps1') - maybe_remove(launcher + '.exe') - if use_bat_file: - write_file(launcher + '.bat', bat_data) - write_file(launcher + '.ps1', ps1_data) - else: - shutil.copyfile(windows_exe, launcher + '.exe') + write_file(launcher + '.bat', bat_data) + write_file(launcher + '.ps1', ps1_data) generate_entry_points(entry_points, os.path.join(__scriptdir__, 'run_python')) generate_entry_points(compiler_entry_points, os.path.join(__scriptdir__, 'run_python_compiler')) if __name__ == '__main__': - sys.exit(main('--all' in sys.argv, '--bat-files' in sys.argv)) + sys.exit(main('--all' in sys.argv)) diff --git a/tools/pylauncher/README.md b/tools/pylauncher/README.md deleted file mode 100644 index 5a5d5fa27c1e8..0000000000000 --- a/tools/pylauncher/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Windows Python Script Launcher - -This directory contains a simple launcher program for Windows which is used to -execute the Emscripten compiler entry points using the python interpreter. It -uses the its own name (the name of the currently running executable) to -determine which python script to run and serves the same purpose as the shell -scripts (see `run_python.sh` ) do on non-Windows platforms. - -## Building - -The primary `build.bat` script uses MSVC and links against `ucrtbase.dll` which -ships as part of the OS since Windows 10 (2015) and is available via -Windows Update for Vista/7/8/8.1. - -`build.sh` cross-compiles with MinGW and links against `msvcrt.dll` (present -on all Windows versions). This script is mostly useful for debugging on Linux. -The .exe file that is checked in here, and used in by emsdk is currently always -built with the `build.bat` script above. - -## Related projects - -### posy-trampoline - -The posy-trampoline project does something similar: - - https://github.com/njsmith/posy/tree/main/src/trampolines/windows-trampolines/posy-trampoline - -However, IIRC it also embeds the target python file onto the executable itself -which is not some Emscripten needs (or wants) to do. Its also written in rust -(which is not something Emscripten has in any of its dependencies yet). -The `uv` tool also embeds a version of this trampiline: - - https://github.com/astral-sh/uv/tree/main/crates/uv-trampoline - -There is also PyInstaller (https://pyinstaller.org/en/stable/), but that seems -to want to wrap up the whole application into the executable too. - -Both these projects seem to change the way the python code itself is delivered -to Windows users. The difference with Emscripten's pylauncher is that -it leaves the layout of the python files unchanged. i.e. Its just a launcher, -not any kind of bundler. This means that Windows users can still see, and even -modify in place, all the python files just like non-Windows users. All the -launcher does is allows the entry point to be an `.exe`. - -The Emscripten launcher is also very small, coming it at just 5.5k at time of -writing. - -The Emscripten launcher also doesn't require any modification of the `.exe` to -deploy it. All one needs to do is copy the unmodified executable alongside a -python file of the same name, and the launcher will find it and run it based -purely on the name of the launcher itself (i.e. argv0). - -### setuptools - -There is windows launcher that is part of setuptools: - - https://github.com/pypa/setuptools/blob/main/launcher.c - -The code is very similar to Emscripten's launcher. In fact, perhaps we should -consider switch it this in the future? - -Unlike Emscripten's laucnher this launcher seems to look for -`-script.py` rather than just `.py`, which is what -the Emscripten laucnher uses. - -This laucnher also seem to examine the `#!` line of the target script and -locate the python executable somehow based on this. diff --git a/tools/pylauncher/build.bat b/tools/pylauncher/build.bat deleted file mode 100644 index e204c642c7a33..0000000000000 --- a/tools/pylauncher/build.bat +++ /dev/null @@ -1,22 +0,0 @@ -:: Build pylauncher.exe using MSVC. -:: -:: This links against ucrtbase.dll (via ucrt.lib) which ships as part of the OS -:: since Windows 10 (2015) and via Windows Update for Vista/7/8/8.1. -:: -:: /O1 : Favor small code (optimization for size) -:: /GS- : Disable buffer security checks (requires vc runtime and not necessary for our tiny command line wrapper) -:: /NODEFAULTLIB : Do not link the default libraries -:: /ENTRY:launcher_main : Use launcher_main() as entry point directly (no CRT startup) -:: /SUBSYSTEM:CONSOLE : Designate as a console app instead of WINDOWS. Needed explicitly because of custom entrypoint. -:: /Brepro : Deterministic (reproducible) output -:: ucrt.lib : Link only against Universal CRT (no vcruntime dependency) - -set OUT=pylauncher.exe -set MACHINE=X64 - -if /i "%~1"=="arm64" ( - set OUT=pylauncher-arm64.exe - set MACHINE=ARM64 -) - -cl pylauncher.c /Fe:%OUT% /O1 /GS- /link /NODEFAULTLIB /ENTRY:launcher_main /SUBSYSTEM:CONSOLE /MACHINE:%MACHINE% /Brepro ucrt.lib kernel32.lib diff --git a/tools/pylauncher/pylauncher-arm64.exe b/tools/pylauncher/pylauncher-arm64.exe deleted file mode 100644 index 88f4b2b87dafd..0000000000000 Binary files a/tools/pylauncher/pylauncher-arm64.exe and /dev/null differ diff --git a/tools/pylauncher/pylauncher.c b/tools/pylauncher/pylauncher.c deleted file mode 100644 index 5a0dcdef9f166..0000000000000 --- a/tools/pylauncher/pylauncher.c +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright 2026 The Emscripten Authors. All rights reserved. - * Emscripten is available under two separate licenses, the MIT license and the - * University of Illinois/NCSA Open Source License. Both these licenses can be - * found in the LICENSE file. - * - * Small win32 application that is used to launcher emscripten via python.exe. - * On non-windows platforms this is done via the run_python.sh shell script. - * - * The binary will look for a python script that matches its own name and run - * that using python.exe. - * - * Built with /NODEFAULTLIB linking only against ucrt.lib (ucrtbase.dll) to - * avoid any dependency on a specific Visual C++ Redistributable version. - */ - -// Define _WIN32_WINNT to Windows 7 for max portability -#define _WIN32_WINNT 0x0601 - -#include -#include -#include -#include -#include -#include - -// ZeroMemory expands to memset which lives in vcruntime, not ucrt. -// SecureZeroMemory is an inline in with no runtime dependency. -#undef ZeroMemory -#define ZeroMemory SecureZeroMemory - -#define WLEN(lit) (sizeof(lit) / sizeof(wchar_t) - 1) - -static bool launcher_debug = false; - -static void dbg(const char* format, ...) { - if (launcher_debug) { - va_list args; - va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); - } -} - -static const wchar_t* get_python_executable() { - const wchar_t* python_exe_w = _wgetenv(L"EMSDK_PYTHON"); - if (!python_exe_w) { - return L"python.exe"; - } - return python_exe_w; -} - -// Get the name of the currently running executable (module) -static wchar_t* get_module_path() { - DWORD buffer_size = MAX_PATH; - wchar_t* module_path_w = malloc(sizeof(wchar_t) * buffer_size); - if (!module_path_w) - abort(); - - DWORD path_len = GetModuleFileNameW(NULL, module_path_w, buffer_size); - // Keep doubling buffer size until GetModuleFileNameW returns something - // less than the full buffer size - while (path_len == buffer_size) { - buffer_size *= 2; - module_path_w = realloc(module_path_w, sizeof(wchar_t) * buffer_size); - if (!module_path_w) - abort(); - path_len = GetModuleFileNameW(NULL, module_path_w, buffer_size); - } - - if (path_len == 0) - abort(); - - return module_path_w; -} - -/** - * A custom replacement for PathGetArgsW that is safe for command lines - * longer than MAX_PATH. - */ -static const wchar_t* find_args(const wchar_t* command_line) { - const wchar_t* p = command_line; - - // Skip past the executable name, which can be quoted. - if (*p == L'"') { - // The path is quoted, find the closing quote. - p++; - while (*p) { - if (*p == L'"') { - p++; - break; - } - p++; - } - } else { - // The path is not quoted, find the first space. - while (*p && *p != L' ' && *p != L'\t') { - p++; - } - } - - // Skip any whitespace between the executable and the first argument. - while (*p && (*p == L' ' || *p == L'\t')) { - p++; - } - - return p; -} - -static bool path_exists(const wchar_t* path) { - return GetFileAttributesW(path) != INVALID_FILE_ATTRIBUTES; -} - -/** - * Create the script path by finding the launcher path and replacing the - * extension with .py. For example `C:\path\to\emcc.exe` becomes - * `C:\path\to\emcc.py`. - * - * If the corresponging .py file does not exist then also look it in the tools - * subdirectory. e.g. `C:\path\to\tools\emcc.py` - */ -static wchar_t* get_script_path() { - wchar_t* script_path = get_module_path(); - if (!script_path) - abort(); - - size_t path_len = wcslen(script_path); - if (path_len < WLEN(L".exe") || _wcsicmp(script_path + path_len - WLEN(L".exe"), L".exe") != 0) - abort(); - // Strip .exe - path_len -= WLEN(L".exe"); - // Append .py (no need to realloc since ".py" is shorter than ".exe") - wcscpy(script_path + path_len, L".py"); - path_len += WLEN(L".py"); - - if (path_exists(script_path)) { - return script_path; - } - - // Python file not found alongside launcher; try under tools - // C:\path\to\emcc.py` => C:\path\to\tools\emcc.py` - size_t dir_len = 0; - for (size_t i = path_len; i > 0; i--) { - if (script_path[i - 1] == L'\\') { - dir_len = i; - break; - } - } - size_t tools_path_size = path_len + WLEN(L"tools\\") + 1; - wchar_t* script_path_tools = malloc(tools_path_size * sizeof(wchar_t)); - swprintf(script_path_tools, tools_path_size, L"%.*lstools\\%ls", (int)dir_len, script_path, script_path + dir_len); - - if (!path_exists(script_path_tools)) { - fprintf(stderr, "pylauncher: target python file not found: %ls / %ls\n", script_path, script_path_tools); - abort(); - } - free(script_path); - - return script_path_tools; -} - -// This gets a name other than main() as a reminder that its return value is not sent anywhere -// (because this file is compiled without a CRT). -void launcher_main() { - // Setting EMCC_LAUNCHER_DEBUG enabled debug output for the launcher itself. - launcher_debug = GetEnvironmentVariableW(L"EMCC_LAUNCHER_DEBUG", NULL, 0); - - dbg("pylauncher: launcher_main\n"); - - const wchar_t* ccache_prefix = L""; - DWORD env_len = GetEnvironmentVariableW(L"_EMCC_CCACHE", NULL, 0); - if (env_len) { - dbg("pylauncher: running via ccache.exe\n"); - ccache_prefix = L"ccache.exe "; - SetEnvironmentVariableW(L"_EMCC_CCACHE", NULL); - } - - const wchar_t* application_name = get_python_executable(); - wchar_t* script_path_w = get_script_path(); - size_t command_line_len = wcslen(ccache_prefix) + wcslen(application_name) + wcslen(script_path_w) + 17; - wchar_t* command_line = malloc(sizeof(wchar_t) * command_line_len); - swprintf(command_line, command_line_len, L"%ls\"%ls\" -E -X utf8 \"%ls\"", ccache_prefix, application_name, script_path_w); - free(script_path_w); - - // -E will not ignore _PYTHON_SYSCONFIGDATA_NAME an internal - // of cpython used in cross compilation via setup.py. - SetEnvironmentVariableW(L"_PYTHON_SYSCONFIGDATA_NAME", NULL); - - // Build the final command line by appending the original arguments - const wchar_t* all_args = find_args(GetCommandLineW()); - if (all_args && *all_args) { - size_t current_len = wcslen(command_line); - size_t args_len = wcslen(all_args); - // +2 for the space and the null terminator - command_line = realloc(command_line, (current_len + args_len + 2) * sizeof(wchar_t)); - if (!command_line) - abort(); - wcscat_s(command_line, current_len + args_len + 2, L" "); - wcscat_s(command_line, current_len + args_len + 2, all_args); - } - - // Work around python bug 34780 by closing stdin, so that it is not inherited - // by the python subprocess. - env_len = GetEnvironmentVariableW(L"EM_WORKAROUND_PYTHON_BUG_34780", NULL, 0); - if (env_len) { - dbg("pylauncher: using EM_WORKAROUND_PYTHON_BUG_34780\n"); - CloseHandle(GetStdHandle(STD_INPUT_HANDLE)); - } - - STARTUPINFOW si; - PROCESS_INFORMATION pi; - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - ZeroMemory(&pi, sizeof(pi)); - - dbg("pylauncher: running: %ls\n", command_line); - if (!CreateProcessW(NULL, command_line, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { - fprintf(stderr, "pylauncher: CreateProcess failed (%lu): %ls\n", GetLastError(), command_line); - abort(); - } - WaitForSingleObject(pi.hProcess, INFINITE); - - DWORD exit_code; - GetExitCodeProcess(pi.hProcess, &exit_code); - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - - dbg("pylauncher: done: %d\n", exit_code); - ExitProcess(exit_code); -} diff --git a/tools/pylauncher/pylauncher.exe b/tools/pylauncher/pylauncher.exe deleted file mode 100644 index 616cd5f44a34f..0000000000000 Binary files a/tools/pylauncher/pylauncher.exe and /dev/null differ