Implement native clang launcher

.circleci/config.yml

@@ -53,6 +53,12 @@ commands:
   bootstrap:
     description: "bootstrap"
     steps:
+      - run:
+          name: Install dependencies (Linux)
+          command: |
+            if command -v apt-get >/dev/null 2>&1; then
+              apt-get install -q -y cmake ninja-build
+            fi
       - run: "$EMSDK_PYTHON ./bootstrap.py"
   pip-install:
     description: "pip install"
@@ -1182,30 +1188,6 @@ jobs:
     steps:
       - test-sockets-chrome
 
-  build-windows-launcher:
-    executor:
-      name: win/server-2022
-      shell: bash.exe -eo pipefail
-    steps:
-      - checkout
-      - run:
-          name: "build pylauncher"
-          shell: cmd.exe
-          command: .circleci\setup_vs2022.bat && cd tools\pylauncher && call build.bat
-      - store_artifacts:
-          path: tools/pylauncher/pylauncher.exe
-          destination: pylauncher.exe
-      - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
-      - run:
-          name: "create_entry_points"
-          command: $EMSDK_PYTHON tools/maint/create_entry_points.py --exe-files
-      - run:
-          name: "crossplatform tests"
-          command: test/runner.exe core0.test_hello_world
-
-
   # windows and mac do not have separate build and test jobs, as they only run
   # a limited set of tests; it is simpler and faster to do it all in one job.
   test-windows:
@@ -1233,10 +1215,6 @@ jobs:
       EMTEST_BROWSER: "0"
     steps:
       - checkout
-      - run:
-          name: Build launcher
-          command: call .circleci\setup_vs2019.bat && cd tools\pylauncher && call build.bat
-          shell: cmd.exe
       - run:
           name: Install packages
           command: |
@@ -1257,9 +1235,7 @@ jobs:
       - upload-test-results
       - run:
           name: "check clean"
-          command: |
-            git checkout tools/pylauncher
-            $EMSDK_PYTHON test/check_clean.py
+          command: $EMSDK_PYTHON test/check_clean.py
 
   test-mac-arm64:
     executor: mac-arm64
@@ -1357,7 +1333,6 @@ workflows:
       - test-node-compat
       - test-windows
       - test-windows-browser-firefox
-      - build-windows-launcher
       - test-mac-arm64:
          requires:
            - build-linux

.github/workflows/ci.yml

@@ -66,6 +66,9 @@ jobs:
             echo "Be sure that you have installed the current emsdk version.  See test/emsdk_version.txt ($(cat test/emsdk_version.txt))."
             exit 1
           fi
+      - name: Check emcc_native generated settings
+        run: |
+          ./tools/emcc_native/gen_settings.py --check
 
   clang-format-diff:
     # This job is disabled until we can make it more precise
@@ -83,22 +86,3 @@ jobs:
         sudo apt-get install clang-format-19
         sudo update-alternatives --install /usr/bin/git-clang-format git-clang-format /usr/bin/git-clang-format-19 100
     - run: tools/maint/clang-format-diff.sh origin/$GITHUB_BASE_REF
-
-  build-pylauncher-arm64:
-    name: Build pylauncher (ARM64)
-    runs-on: windows-11-arm
-    steps:
-    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-    - name: Setup MSVC
-      uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
-      with:
-        arch: arm64
-    - name: Build pylauncher
-      run: |
-        cd tools\pylauncher
-        call build.bat arm64
-      shell: cmd
-    - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-      with:
-        name: pylauncher-arm64.exe
-        path: tools/pylauncher/pylauncher-arm64.exe

Makefile

@@ -13,10 +13,18 @@ install:
 	./tools/install.py $(DESTDIR)
 	npm install --omit=dev --prefix $(DESTDIR)
 
+out/build_emcc_native/native_tests emcc_native:
+	cmake -B out/build_emcc_native -S tools/emcc_native -G Ninja -DCMAKE_BUILD_TYPE=Release
+	cmake --build out/build_emcc_native --config Release
+	cmake --install out/build_emcc_native --config Release
+
+test: out/build_emcc_native/native_tests
+	ctest --test-dir out/build_emcc_native --output-on-failure
+
 # Create an distributable archive of emscripten suitable for use
 # by end users. This archive excludes node_modules as it can include native
 # modules which can't be safely pre-packaged.
 $(DISTFILE): install
 	tar cf $@ --exclude=node_modules -C `dirname $(DESTDIR)` `basename $(DESTDIR)`
 
-.PHONY: dist install
+.PHONY: dist install emcc_native test

README.md

@@ -39,11 +39,11 @@ There are two primary ways to install Emscripten:
     The easiest way to get started is by using the Emscripten SDK. Follow the instructions on the [downloads page](https://emscripten.org/docs/getting_started/downloads.html) to install it.
 
 2.  **From a Git Checkout (Manual Installation)**
-    If you have cloned the repository from Git, you can install the dependencies manually and then run the bootstrap script:
+    If you have cloned the repository from Git, run `bootstrap.py` to build the native launcher (`emcc_native`) and set up dependencies:
     ```bash
     ./bootstrap.py
     ```
-    For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html).
+    Building `emcc_native` requires CMake 3.20+ and a C++20 host compiler toolchain. Alternatively, setting `EMCC_NATIVE=0` in your environment before running `./bootstrap.py` will generate legacy Python launcher scripts (e.g. `.bat` / `.ps1` files on Windows) via `./tools/maint/create_entry_points.py`. For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html).
 
 ## Using the compiler
 

bootstrap.py

@@ -95,18 +95,22 @@ def run_cmd(cmd):
   subprocess.run(cmd, check=True, text=True, encoding='utf-8', cwd=utils.path_from_root())
 
 
+def build_emcc_native():
+  build_dir = utils.path_from_root('out/build_emcc_native')
+  source_dir = utils.path_from_root('tools/emcc_native')
+  cmd = ['cmake', '-B', build_dir, source_dir, '-DCMAKE_BUILD_TYPE=Release']
+  if not utils.WINDOWS and shutil.which('ninja'):
+    cmd.extend(['-G', 'Ninja'])
+  run_cmd(cmd)
+  run_cmd(['cmake', '--build', build_dir, '--config', 'Release'])
+  run_cmd(['cmake', '--install', build_dir, '--config', 'Release'])
+
+
 actions = [
   ('npm packages', [
      'package.json',
      'package-lock.json',
    ], ['npm', 'ci']),
-  ('create entry points', [
-     'tools/maint/create_entry_points.py',
-     'tools/pylauncher/pylauncher.exe',
-     'tools/maint/run_python.bat',
-     'tools/maint/run_python.sh',
-     'tools/maint/run_python.ps1',
-   ], [sys.executable, 'tools/maint/create_entry_points.py']),
   ('git submodules', [
      'test/third_party/posixtestsuite/',
      'test/third_party/googletest',
@@ -118,6 +122,28 @@ def run_cmd(cmd):
    ], maybe_install_hooks),
 ]
 
+if os.environ.get('EMCC_NATIVE') == '0':
+  actions.append(('legacy entry points', [
+     'tools/maint/create_entry_points.py',
+     'tools/maint/run_python.sh',
+     'tools/maint/run_python.bat',
+     'tools/maint/run_python.ps1',
+     'tools/maint/run_python_compiler.sh',
+     'tools/maint/run_python_compiler.bat',
+     'tools/maint/run_python_compiler.ps1',
+   ], [sys.executable, utils.path_from_root('tools/maint/create_entry_points.py')]))
+else:
+  actions.append(('build emcc_native', [
+     'tools/emcc_native/CMakeLists.txt',
+     'tools/emcc_native/main.cpp',
+     'tools/emcc_native/driver.cpp',
+     'tools/emcc_native/driver.h',
+     'tools/emcc_native/exec.cpp',
+     'tools/emcc_native/exec.h',
+     'tools/emcc_native/config.cpp',
+     'tools/emcc_native/config.h',
+   ], build_emcc_native))
+
 
 def main(args):
   parser = argparse.ArgumentParser(description=__doc__)

docs/design/03-native-clang-frontend.md

@@ -1,6 +1,6 @@
 # Design Doc: Native Launcher / Clang Frontend
 
-- **Status**: Draft
+- **Status**: Phase 1 Completed
 - **Bug**: https://github.com/emscripten-core/emscripten/issues/26453
 
 ## Context

site/source/docs/building_from_source/index.rst

@@ -7,11 +7,18 @@ Building Emscripten from Source
 Building Emscripten yourself is an alternative to getting binaries using the
 emsdk.
 
-Emscripten itself is written in Python and JavaScript so it does not need to be
-compiled.  However, after checkout you will need to run the top level
-``bootstrap.py`` script before the toolchain is usable.  This performs
-various steps including ``npm install`` and the creation of compiler entry
-points (e.g. `.bat` files on windows).
+Emscripten itself is primarily written in Python and JavaScript. However,
+after checkout you will need to run the top-level ``bootstrap.py`` script
+before the toolchain is usable. This performs various steps including ``npm
+install`` and building the native compiler frontend launcher (``emcc_native``),
+which provides high-performance ``emcc`` and ``em++`` binaries.
+
+Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler
+toolchain (which you already need for building LLVM and Binaryen). If you prefer
+not to perform a native build, setting ``EMCC_NATIVE=0`` in your environment
+before running ``./bootstrap.py`` instructs it to generate legacy Python launcher
+scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via
+``./tools/maint/create_entry_points.py``.
 
 Emscripten comes with its own versions of some C/C++ system libraries which
 ``emcc`` builds automatically as and when needed (in the emsdk builds, these are

site/source/docs/building_from_source/toolchain_what_is_needed.rst

@@ -39,7 +39,9 @@ In general a complete Emscripten environment requires the following tools. First
 Compiler toolchain
 ------------------
 
-When building LLVM and Binaryen from source code, whether "manually" or using the SDK, you will need a *compiler toolchain*:
+When building Emscripten from source (via ``bootstrap.py`` to build
+``emcc_native``), or when building LLVM and Binaryen from source, you will need
+a C++20 host *compiler toolchain* and CMake 3.20+:
 
 - Windows: You will need `Visual Studio <https://visualstudio.microsoft.com/>`_ (2019 or above) and `cmake <http://www.cmake.org/cmake/resources/software.html>`_ (3.20 or above).
 

site/source/docs/contributing/developers_guide.rst

@@ -16,10 +16,22 @@ interested in helping out!
 Setting up
 ==========
 
-For contributing to core Emscripten code, such as ``emcc.py``, you don't need to
-build any binaries as ``emcc.py`` is in Python, and the core JS generation is
-in JavaScript. You do still need binaries for LLVM and Binaryen, which you can
-get using the emsdk.
+When setting up a Git checkout of Emscripten, run the top-level ``bootstrap.py``
+script to set up dependencies (such as ``npm install``) and build the native
+compiler frontend launcher (``emcc_native``):
+
+::
+
+    ./bootstrap.py
+
+Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler
+toolchain. If you prefer not to build the native launcher, setting ``EMCC_NATIVE=0``
+in your environment before running ``./bootstrap.py`` will generate legacy Python
+launcher scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via
+``./tools/maint/create_entry_points.py``.
+
+For LLVM and Binaryen binaries, you don't need to build them from source if you
+are only contributing to Emscripten; you can get them using the emsdk.
 
 If you want to contribute back to Emscripten, it is recommended that you install
 the precise version of the emsdk binaries that are used by Emscripten CI when

test/test_other.py

@@ -12279,9 +12279,9 @@ def test_xclang_flag(self):
     self.run_process([EMCC, '-c', '-o', 'out.o', '-Xclang', '-include', '-Xclang', 'foo.h', test_file('hello_world.c')])
 
   def test_emcc_size_parsing(self):
-    create_file('foo.h', ' ')
-    self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.h'], 'error: invalid byte size `X`.  Valid suffixes are: kb, mb, gb, tb')
-    self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.h'], 'error: invalid byte size `11PB`.  Valid suffixes are: kb, mb, gb, tb')
+    create_file('foo.c', ' ')
+    self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.c'], 'error: invalid byte size `X`.  Valid suffixes are: kb, mb, gb, tb')
+    self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.c'], 'error: invalid byte size `11PB`.  Valid suffixes are: kb, mb, gb, tb')
 
   def test_native_call_before_init(self):
     self.set_setting('ASSERTIONS')

tools/compile.py

@@ -16,6 +16,9 @@
 
 get_cflags(): In addition to compiler flags this function also returns pre-processor
 flags. For example, include paths and macro definitions.
+
+NOTE: Default compiler flag construction logic here is also implemented natively
+in tools/emcc_native/driver.cpp. Keep changes in sync between both places!
 """
 
 import os