diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..809f2b5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,110 @@ +name: CI + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-mingw: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - target_arch: win32 + run_tests: true + - target_arch: x64 + run_tests: true + + steps: + - uses: actions/checkout@v4 + + - name: Setup MSYS2 toolchains + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + git + make + mingw-w64-x86_64-gcc + mingw-w64-i686-gcc + mingw-w64-x86_64-clang + mingw-w64-clang-aarch64-gcc + + - name: Build patched LuaJIT (MinGW) + shell: msys2 {0} + run: HOST_CC=gcc TARGET_ARCH=${{ matrix.target_arch }} ./scripts/build-mingw.sh + + - name: Run UTF-8 wrapper tests (MinGW) + if: matrix.run_tests + shell: msys2 {0} + run: | + if [ "${{ matrix.target_arch }}" = "win32" ]; then + export PATH="/mingw32/bin:$PATH" + else + export PATH="/mingw64/bin:$PATH" + fi + IAT_TEST_VAR=Ελλ_中文_한국_عربي_кирил_деवनागरी \ + LUAJIT_DLL=.work/LuaJIT/src/lua51.dll \ + ./.work/LuaJIT/src/luajit.exe tests/test_utf8_wrappers.lua + + - name: Upload MinGW artifacts + uses: actions/upload-artifact@v4 + with: + name: luajit-mingw-${{ matrix.target_arch }} + path: | + .work/LuaJIT/src/luajit.exe + .work/LuaJIT/src/lua51.dll + .work/LuaJIT/src/lua51.lib + + build-msvc: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - target_arch: win32 + run_tests: true + - target_arch: x64 + run_tests: true + - target_arch: arm64 + run_tests: false + + steps: + - uses: actions/checkout@v4 + + - name: Resolve Visual Studio installation + shell: pwsh + run: | + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vsPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if (-not $vsPath) { throw "Visual Studio installation path not found" } + "VSINSTALLDIR=$vsPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Build patched LuaJIT (MSVC) + shell: cmd + run: | + set TARGET_ARCH=${{ matrix.target_arch }} + scripts\build-msvc.bat + + - name: Run UTF-8 wrapper tests (MSVC) + if: matrix.run_tests + shell: cmd + run: | + chcp 65001 >nul + set IAT_TEST_VAR=Ελλ_中文_한국_عربي_кирил_देवनागरी + set LUAJIT_DLL=.work\LuaJIT\src\lua51.dll + .work\LuaJIT\src\luajit.exe tests\test_utf8_wrappers.lua + + - name: Upload MSVC artifacts + uses: actions/upload-artifact@v4 + with: + name: luajit-msvc-${{ matrix.target_arch }} + path: | + .work/LuaJIT/src/luajit.exe + .work/LuaJIT/src/lua51.dll + .work/LuaJIT/src/lua51.lib diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..60795fd --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,145 @@ +name: Release + +on: + workflow_dispatch: + inputs: + luajit_ref: + description: "LuaJIT git ref (tag/branch/commit)" + required: true + default: "v2.1" + release_tag: + description: "Release tag to publish" + required: true + release_name: + description: "Release title" + required: false + default: "LuaJIT Unicode build" + push: + tags: + - "release-*" + +permissions: + contents: read + +jobs: + build: + runs-on: windows-latest + env: + LUAJIT_REF: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.luajit_ref || 'v2.1' }} + strategy: + fail-fast: false + matrix: + include: + - toolchain: mingw + target_arch: win32 + run_tests: true + - toolchain: mingw + target_arch: x64 + run_tests: true + - toolchain: msvc + target_arch: win32 + run_tests: true + - toolchain: msvc + target_arch: x64 + run_tests: true + - toolchain: msvc + target_arch: arm64 + run_tests: false + + steps: + - uses: actions/checkout@v4 + + - name: Setup MSYS2 toolchains (MinGW) + if: matrix.toolchain == 'mingw' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + git + make + mingw-w64-x86_64-gcc + mingw-w64-i686-gcc + mingw-w64-x86_64-clang + mingw-w64-clang-aarch64-gcc + + - name: Resolve Visual Studio installation (MSVC) + if: matrix.toolchain == 'msvc' + shell: pwsh + run: | + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vsPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if (-not $vsPath) { throw "Visual Studio installation path not found" } + "VSINSTALLDIR=$vsPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Build patched LuaJIT (MinGW) + if: matrix.toolchain == 'mingw' + shell: msys2 {0} + run: | + HOST_CC=gcc \ + LUAJIT_REF=$LUAJIT_REF \ + TARGET_ARCH=${{ matrix.target_arch }} \ + ./scripts/build-mingw.sh + + - name: Build patched LuaJIT (MSVC) + if: matrix.toolchain == 'msvc' + shell: cmd + run: | + set LUAJIT_REF=%LUAJIT_REF% + set TARGET_ARCH=${{ matrix.target_arch }} + scripts\build-msvc.bat + + - name: Run UTF-8 wrapper tests (MinGW) + if: matrix.toolchain == 'mingw' && matrix.run_tests + shell: msys2 {0} + run: | + if [ "${{ matrix.target_arch }}" = "win32" ]; then + export PATH="/mingw32/bin:$PATH" + else + export PATH="/mingw64/bin:$PATH" + fi + IAT_TEST_VAR=Ελλ_中文_한국_عربي_кирил_देवनागरी \ + LUAJIT_DLL=.work/LuaJIT/src/lua51.dll \ + ./.work/LuaJIT/src/luajit.exe tests/test_utf8_wrappers.lua + + - name: Run UTF-8 wrapper tests (MSVC) + if: matrix.toolchain == 'msvc' && matrix.run_tests + shell: cmd + run: | + chcp 65001 >nul + set IAT_TEST_VAR=Ελλ_中文_한국_عربي_кирил_देवनागरी + set LUAJIT_DLL=.work\LuaJIT\src\lua51.dll + .work\LuaJIT\src\luajit.exe tests\test_utf8_wrappers.lua + + - name: Package artifacts + shell: pwsh + run: | + New-Item -ItemType Directory -Path out -Force | Out-Null + $prefix = "luajit-unicode-${{ matrix.toolchain }}-${{ matrix.target_arch }}" + Copy-Item .work\LuaJIT\src\luajit.exe "out\$prefix-luajit.exe" + if (Test-Path .work\LuaJIT\src\lua51.dll) { Copy-Item .work\LuaJIT\src\lua51.dll "out\$prefix-lua51.dll" } + if (Test-Path .work\LuaJIT\src\lua51.lib) { Copy-Item .work\LuaJIT\src\lua51.lib "out\$prefix-lua51.lib" } + + - name: Upload workflow artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.toolchain }}-${{ matrix.target_arch }} + path: out/* + + publish-release: + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist + + - name: Publish GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }} + name: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_name || format('LuaJIT Unicode build {0}', github.ref_name) }} + files: dist/*/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d9bad5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.work/ +tests/_unicode_fixture/ diff --git a/README.md b/README.md index 7d3bb07..b622bb6 100644 --- a/README.md +++ b/README.md @@ -3,21 +3,162 @@ This is an adaptation of https://github.com/Lekensteyn/lua-unicode to work under LuaJIT and for the purpose of **Pragtical Code Editor**. -## Changes +## Build integration without editing LuaJIT sources manually -1. Copy `src/utf8_wrappers.h` and `src/utf8_wrappers.c` to LuaJIT `src` dir. -2. Patch `src/luaconf.h` to add the following in the *Local configuration* part: +This repository now provides standalone scripts that: -```c -#if defined(LUA_LIB) && (defined(lib_aux_c) || defined(lib_io_c) || \ - defined(lib_package_c) || defined(lib_os_c)) -#include "utf8_wrappers.h" -#endif +1. download LuaJIT sources automatically; +2. copy `utf8_wrappers.{c,h}` into LuaJIT `src/`; +3. force-include `utf8_wrappers.h` via compiler flags; +4. apply minimal patch files only where extra objects need to be added; +5. run LuaJIT native build scripts for each toolchain. + +## Implementation options analysis + +### Option 1: Keep manual patching (old approach) +- **Pros:** trivial maintenance in this repository. +- **Cons:** easy to make mistakes, not reproducible, requires hand-editing LuaJIT sources every time. + +### Option 2: Override everything only with compiler flags +- **Pros:** no source patch files. +- **Cons:** brittle for LuaJIT internals, hard to inject `utf8_wrappers.c` into both GNU Make and `msvcbuild.bat` flows, harder to support all targets consistently. + +### Option 3 (implemented): Hybrid wrapper (forced include + minimal patches) +- **Pros:** reproducible, avoids patching `luaconf.h`, keeps patch scope smaller, supports separate MinGW/MSVC workflows. +- **Cons:** object list integration is still patched in upstream build scripts. + +## Files + +- `patches/luajit-makefile-unicode.patch` (MinGW/GNU Make flow) +- `patches/luajit-msvcbuild-unicode.patch` (MSVC flow) +- `scripts/build-mingw.sh` +- `scripts/build-msvc.bat` + +## Imported-symbol audit (Windows) + +Based on the current LuaJIT import list, the path/command/environment APIs that +need UTF-8 adaptation are already covered in `utf8_wrappers`: + +- `fopen` / `freopen` +- `remove` +- `rename` +- `_popen` +- `system` +- `getenv` +- `GetModuleFileNameA` +- `LoadLibraryExA` + +Most other imported symbols from the list are math/runtime/threading/memory APIs +or descriptor-level file APIs (`_write`, `_lseeki64`, `_fileno`, etc.) that do +not accept path strings and therefore do not need Unicode path wrappers. + +## MinGW usage + +```bash +TARGET_ARCH=x64 ./scripts/build-mingw.sh +``` + +Ready-to-run commands for each target: + +```bash +# Win32 +TARGET_ARCH=win32 ./scripts/build-mingw.sh + +# x64 +TARGET_ARCH=x64 ./scripts/build-mingw.sh + +# ARM64 +TARGET_ARCH=arm64 ./scripts/build-mingw.sh +``` + +Supported `TARGET_ARCH` values: +- `win32` (i686 toolchain) +- `x64` (x86_64 toolchain) +- `arm64` (aarch64 toolchain) + +Useful variables: +- `LUAJIT_REF` (default: `v2.1`) +- `WORK_DIR` (default: `.work`) +- `LUAJIT_DIR` (default: `$WORK_DIR/LuaJIT`) +- `PREPARE_ONLY=1` (download + patch, no build) + +## MSVC usage + +Open the appropriate Visual Studio command prompt for target architecture +(`x86`, `x64`, or `arm64`) and run: + +```bat +set TARGET_ARCH=x64 +scripts\build-msvc.bat ``` -3. Patch `src/Makefile` to compile `src/utf8_wrappers.c`: +Ready-to-run commands for each target: -```diff -- lib_buffer.o -+ lib_buffer.o utf8_wrappers.o +```bat +:: Win32 +set TARGET_ARCH=win32 +scripts\build-msvc.bat + +:: x64 +set TARGET_ARCH=x64 +scripts\build-msvc.bat + +:: ARM64 +set TARGET_ARCH=arm64 +scripts\build-msvc.bat ``` + +Supported `TARGET_ARCH` values: +- `win32` +- `x64` +- `arm64` + +Useful variables: +- `LUAJIT_REF` (default: `v2.1`) +- `WORK_DIR` (default: `.work`) +- `LUAJIT_DIR` (default: `%WORK_DIR%\LuaJIT`) +- `PREPARE_ONLY=1` (download + patch, no build) + +## CI and releases + +- PR CI builds patched LuaJIT on Windows for MinGW/MSVC and runs UTF-8 wrapper tests + for runnable targets (`win32`, `x64`). +- MinGW ARM64 CI job is temporarily disabled due to current upstream LuaJIT ARM64 build + incompatibility in this cross-build runner environment. +- MSVC ARM64 runtime tests are currently skipped in CI. +- Release workflow can be started manually (`workflow_dispatch`) with: + - `luajit_ref` (LuaJIT ref to build from), + - `release_tag` (tag for the GitHub Release), + - `release_name` (optional release title). + +Test file used in CI: +- `tests/test_utf8_wrappers.lua` +- Fixture setup script: `tests/test_setup.cmd` + +### ARM/ARM64 build-and-test feasibility (analysis only) + +- **MSVC ARM64 build:** works on `windows-latest` using `vcvarsall amd64_arm64`. +- **MSVC ARM64 runtime tests:** require reliable execution policy for ARM64 artifacts + on the runner image and stable UTF-8 console/environment behavior. Kept disabled + for now in CI to avoid blocking unrelated PRs. +- **MinGW ARM64 build:** can be driven with host clang cross-targeting + (`--target=aarch64-w64-windows-gnu`) when x86_64 clang tools are installed. +- **MinGW ARM64 runtime tests:** not enabled yet; practical coverage needs either a + native ARM64 Windows runner (preferred) or a validated emulation strategy. + +## Upstream hook proposal (to remove local patches later) + +Current LuaJIT build files do not expose extension points for adding external C +objects cleanly. Minimal upstream-compatible hooks that would allow patch-free integration: + +1. `src/Makefile`: + - add `EXTRA_TARGET_CFLAGS ?=` + - add `EXTRA_LJLIB_O ?=` + - append them to `TARGET_CFLAGS` and `LJLIB_O`. +2. `src/msvcbuild.bat`: + - add optional env vars like `LJ_EXTRA_CFILES`, `LJ_EXTRA_OBJS`, `LJ_EXTRA_CFLAGS` + - append them in compile/link commands. +3. Keep default behavior unchanged when these vars are not set. + +With these hooks, this repository could switch to pure wrapper scripts without any +content patches to upstream files. diff --git a/patches/luajit-makefile-unicode.patch b/patches/luajit-makefile-unicode.patch new file mode 100644 index 0000000..f96dcc0 --- /dev/null +++ b/patches/luajit-makefile-unicode.patch @@ -0,0 +1,11 @@ +--- a/src/Makefile ++++ b/src/Makefile +@@ -511,7 +511,7 @@ + + LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \ + lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o \ +- lib_buffer.o ++ lib_buffer.o utf8_wrappers.o + LJLIB_C= $(LJLIB_O:.o=.c) + + LJCORE_O= lj_assert.o lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \ diff --git a/patches/luajit-msvcbuild-unicode.patch b/patches/luajit-msvcbuild-unicode.patch new file mode 100644 index 0000000..2f47fa4 --- /dev/null +++ b/patches/luajit-msvcbuild-unicode.patch @@ -0,0 +1,43 @@ +--- a/src/msvcbuild.bat ++++ b/src/msvcbuild.bat +@@ -110,32 +110,32 @@ + @set LJLINK=%LJLINK% %LJLINKTYPE% %LJLINKTARGET% + @if "%1"=="amalg" goto :AMALGDLL + @if "%1"=="static" goto :STATIC +-%LJCOMPILE% %LJDYNBUILD% lj_*.c lib_*.c ++%LJCOMPILE% %LJDYNBUILD% lj_*.c lib_*.c utf8_wrappers.c + @if errorlevel 1 goto :BAD + @if "%1"=="mixed" goto :STATICLIB +-%LJLINK% /DLL /OUT:%LJDLLNAME% lj_*.obj lib_*.obj ++%LJLINK% /DLL /OUT:%LJDLLNAME% lj_*.obj lib_*.obj utf8_wrappers.obj legacy_stdio_definitions.lib ucrt.lib + @if errorlevel 1 goto :BAD + @goto :MTDLL + :STATIC +-%LJCOMPILE% lj_*.c lib_*.c ++%LJCOMPILE% lj_*.c lib_*.c utf8_wrappers.c + @if errorlevel 1 goto :BAD + :STATICLIB +-%LJLIB% /OUT:%LJLIBNAME% lj_*.obj lib_*.obj ++%LJLIB% /OUT:%LJLIBNAME% lj_*.obj lib_*.obj utf8_wrappers.obj + @if errorlevel 1 goto :BAD + @goto :MTDLL + :AMALGDLL + @if "%2"=="static" goto :AMALGSTATIC +-%LJCOMPILE% %LJDYNBUILD% ljamalg.c ++%LJCOMPILE% %LJDYNBUILD% ljamalg.c utf8_wrappers.c + @if errorlevel 1 goto :BAD + @if "%2"=="mixed" goto :AMALGSTATICLIB +-%LJLINK% /DLL /OUT:%LJDLLNAME% ljamalg.obj lj_vm.obj ++%LJLINK% /DLL /OUT:%LJDLLNAME% ljamalg.obj utf8_wrappers.obj lj_vm.obj legacy_stdio_definitions.lib ucrt.lib + @if errorlevel 1 goto :BAD + @goto :MTDLL + :AMALGSTATIC +-%LJCOMPILE% ljamalg.c ++%LJCOMPILE% ljamalg.c utf8_wrappers.c + @if errorlevel 1 goto :BAD + :AMALGSTATICLIB +-%LJLIB% /OUT:%LJLIBNAME% ljamalg.obj lj_vm.obj ++%LJLIB% /OUT:%LJLIBNAME% ljamalg.obj utf8_wrappers.obj lj_vm.obj + @if errorlevel 1 goto :BAD + :MTDLL + if exist %LJDLLNAME%.manifest^ diff --git a/scripts/build-mingw.sh b/scripts/build-mingw.sh new file mode 100755 index 0000000..9c28fec --- /dev/null +++ b/scripts/build-mingw.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK_DIR="${WORK_DIR:-$ROOT_DIR/.work}" +LUAJIT_DIR="${LUAJIT_DIR:-$WORK_DIR/LuaJIT}" +LUAJIT_REF="${LUAJIT_REF:-v2.1}" +TARGET_ARCH="${TARGET_ARCH:-x64}" + +case "$TARGET_ARCH" in + win32) + export PATH="/mingw32/bin:$PATH" + # Use unprefixed MinGW tools selected by PATH. + CROSS="${CROSS:-}" + CC_BIN="${CC:-gcc}" + HOST_CC="${HOST_CC:-gcc -m32}" + TARGET_FLAGS_DEFAULT="" + ;; + x64) + export PATH="/mingw64/bin:$PATH" + # Use unprefixed MinGW tools selected by PATH. + CROSS="${CROSS:-}" + CC_BIN="${CC:-gcc}" + HOST_CC="${HOST_CC:-gcc}" + TARGET_FLAGS_DEFAULT="" + ;; + arm64) + export PATH="/mingw64/bin:$PATH" + CROSS="${CROSS:-}" + # Use x86_64 host-executable clang with explicit arm64 Windows target flags. + CC_BIN="${CC:-clang}" + HOST_CC="${HOST_CC:-gcc}" + TARGET_FLAGS_DEFAULT="--target=aarch64-w64-windows-gnu -DLUAJIT_NUMMODE=2" + ;; + *) + echo "Unsupported TARGET_ARCH: $TARGET_ARCH (expected: win32, x64, arm64)" >&2 + exit 1 + ;; +esac + +mkdir -p "$WORK_DIR" +if [ ! -d "$LUAJIT_DIR/.git" ]; then + git clone --depth 1 --branch "$LUAJIT_REF" https://github.com/LuaJIT/LuaJIT "$LUAJIT_DIR" +fi + +cp "$ROOT_DIR/src/utf8_wrappers.c" "$LUAJIT_DIR/src/utf8_wrappers.c" +cp "$ROOT_DIR/src/utf8_wrappers.h" "$LUAJIT_DIR/src/utf8_wrappers.h" + +TARGET_CC_BIN="${CROSS}${CC_BIN}" +if ! command -v "$TARGET_CC_BIN" >/dev/null 2>&1; then + echo "Compiler not found: $TARGET_CC_BIN" >&2 + exit 1 +fi + +if [ -n "$CROSS" ] && command -v "${CROSS}strip" >/dev/null 2>&1; then + TARGET_STRIP_BIN="${CROSS}strip" +elif command -v strip >/dev/null 2>&1; then + TARGET_STRIP_BIN="strip" +elif command -v llvm-strip >/dev/null 2>&1; then + TARGET_STRIP_BIN="llvm-strip" +else + echo "Error: strip tool not found (tried: ${CROSS}strip, strip, llvm-strip)." >&2 + exit 1 +fi + +if ! grep -Eq 'lib_buffer\.o[[:space:]]+utf8_wrappers\.o' "$LUAJIT_DIR/src/Makefile"; then + perl -0777 -i -pe 's/(lib_buffer\.o)([ \t]*\r?\nLJLIB_C=)/$1 utf8_wrappers.o$2/' "$LUAJIT_DIR/src/Makefile" + if ! grep -Eq 'lib_buffer\.o[[:space:]]+utf8_wrappers\.o' "$LUAJIT_DIR/src/Makefile"; then + echo "Failed to update LuaJIT src/Makefile with utf8_wrappers.o" >&2 + exit 1 + fi +fi + +if [ "${PREPARE_ONLY:-0}" = "1" ]; then + echo "Prepared LuaJIT sources in $LUAJIT_DIR" + exit 0 +fi + +# Avoid leaking external TARGET_ARCH env into LuaJIT Makefile internals. +make -C "$LUAJIT_DIR/src" \ + CC="$CC_BIN" \ + HOST_CC="$HOST_CC" \ + CROSS="$CROSS" \ + TARGET_FLAGS="${TARGET_FLAGS:-$TARGET_FLAGS_DEFAULT}" \ + TARGET_SYS=Windows \ + TARGET_CFLAGS="${TARGET_CFLAGS:-} -include utf8_wrappers.h" \ + TARGET_STRIP="$TARGET_STRIP_BIN" \ + TARGET_ARCH= \ + "$@" diff --git a/scripts/build-msvc.bat b/scripts/build-msvc.bat new file mode 100644 index 0000000..a429fa2 --- /dev/null +++ b/scripts/build-msvc.bat @@ -0,0 +1,103 @@ +@echo off +setlocal + +set "ROOT_DIR=%~dp0.." +for %%I in ("%ROOT_DIR%") do set "ROOT_DIR=%%~fI" + +if "%WORK_DIR%"=="" set "WORK_DIR=%ROOT_DIR%\.work" +if "%LUAJIT_DIR%"=="" set "LUAJIT_DIR=%WORK_DIR%\LuaJIT" +if "%LUAJIT_REF%"=="" set "LUAJIT_REF=v2.1" + +if "%TARGET_ARCH%"=="" set "TARGET_ARCH=x64" +if /I not "%TARGET_ARCH%"=="win32" if /I not "%TARGET_ARCH%"=="x64" if /I not "%TARGET_ARCH%"=="arm64" ( + echo Unsupported TARGET_ARCH: %TARGET_ARCH% ^(expected: win32, x64, arm64^) + exit /b 1 +) +if /I "%TARGET_ARCH%"=="win32" set "VS_ARCH=x86" +if /I "%TARGET_ARCH%"=="x64" set "VS_ARCH=x64" +if /I "%TARGET_ARCH%"=="arm64" set "VS_ARCH=arm64" +if /I "%TARGET_ARCH%"=="win32" set "VCVARS_ARCH=x86" +if /I "%TARGET_ARCH%"=="x64" set "VCVARS_ARCH=x64" +if /I "%TARGET_ARCH%"=="arm64" set "VCVARS_ARCH=amd64_arm64" + +if not exist "%WORK_DIR%" mkdir "%WORK_DIR%" +if not exist "%LUAJIT_DIR%\.git" ( + git clone --depth 1 --branch "%LUAJIT_REF%" https://github.com/LuaJIT/LuaJIT "%LUAJIT_DIR%" + if errorlevel 1 exit /b 1 +) + +copy /Y "%ROOT_DIR%\src\utf8_wrappers.c" "%LUAJIT_DIR%\src\utf8_wrappers.c" >nul +if errorlevel 1 exit /b 1 +copy /Y "%ROOT_DIR%\src\utf8_wrappers.h" "%LUAJIT_DIR%\src\utf8_wrappers.h" >nul +if errorlevel 1 exit /b 1 + +call :APPLY_PATCH "%ROOT_DIR%\patches\luajit-msvcbuild-unicode.patch" +if errorlevel 1 exit /b 1 + +if "%PREPARE_ONLY%"=="1" ( + echo Prepared LuaJIT sources in %LUAJIT_DIR% + exit /b 0 +) + +if not defined VSINSTALLDIR call :RESOLVE_VSINSTALLDIR +if errorlevel 1 exit /b 1 +if not "%VSINSTALLDIR:~-1%"=="\" set "VSINSTALLDIR=%VSINSTALLDIR%\" + +set "VCVARSALL=%VSINSTALLDIR%VC\Auxiliary\Build\vcvarsall.bat" +if not exist "%VCVARSALL%" ( + echo Failed to locate vcvarsall.bat under VSINSTALLDIR=%VSINSTALLDIR% + exit /b 1 +) + +call "%VCVARSALL%" %VCVARS_ARCH% +if errorlevel 1 exit /b 1 +where cl >nul 2>&1 +if errorlevel 1 ( + echo MSVC compiler tools are not available after environment initialization. + exit /b 1 +) + +rem Force wrapper header for all C translation units compiled by msvcbuild.bat. +rem Wrapper macros only activate in lib_* units where corresponding defines are present. +set "CL=/FIutf8_wrappers.h /I. %CL%" + +pushd "%LUAJIT_DIR%\src" +call msvcbuild.bat %* +set "BUILD_RC=%ERRORLEVEL%" +popd + +exit /b %BUILD_RC% + +:APPLY_PATCH +set "PATCH_FILE=%~1" +git -C "%LUAJIT_DIR%" apply --reverse --check "%PATCH_FILE%" >nul 2>&1 +if not errorlevel 1 ( + echo Patch already applied: %~nx1 + exit /b 0 +) + +git -C "%LUAJIT_DIR%" apply "%PATCH_FILE%" +if errorlevel 1 ( + echo Failed to apply patch: %PATCH_FILE% + exit /b 1 +) +exit /b 0 + +:RESOLVE_VSINSTALLDIR +set "VSW=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +if not exist "%VSW%" ( + echo vswhere.exe not found: %VSW% + exit /b 1 +) +set "VS_PATH=" +for /f "usebackq tokens=*" %%i in (`"%VSW%" -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + set "VS_PATH=%%i" +) +if "%VS_PATH%"=="" ( + echo Unable to locate Visual Studio installation with VC tools ^ +(vswhere returned no matching path). + exit /b 1 +) +set "VSINSTALLDIR=%VS_PATH%" +if not "%VSINSTALLDIR:~-1%"=="\" set "VSINSTALLDIR=%VSINSTALLDIR%\" +exit /b 0 diff --git a/tests/test_setup.cmd b/tests/test_setup.cmd new file mode 100644 index 0000000..904cf11 --- /dev/null +++ b/tests/test_setup.cmd @@ -0,0 +1,17 @@ +@echo off +setlocal +chcp 65001 >nul + +set "DIR=tests\_unicode_fixture" + +rmdir /s /q "%DIR%" 2>nul +if exist "%DIR%" exit /b 1 +mkdir "%DIR%" || exit /b 1 + +rem Unicode scripts in fixture name: Greek, Chinese, Korean, Arabic, Cyrillic, Devanagari. +powershell -NoProfile -ExecutionPolicy Bypass -Command ^ + "$ErrorActionPreference='Stop'; $dir='tests/_unicode_fixture'; $codes=@(0x395,0x3bb,0x3bb,0x5f,0x4e2d,0x6587,0x5f,0xd55c,0xad6d,0x5f,0x639,0x631,0x628,0x64a,0x5f,0x43a,0x438,0x440,0x438,0x43b,0x5f,0x926,0x947,0x935,0x928,0x93e,0x917,0x930,0x940); $name=-join ($codes|ForEach-Object {[char]$_}); $base=Join-Path $dir $name; Set-Content -LiteralPath ($base+'.txt') -Value 'fixture-content' -Encoding utf8; Set-Content -LiteralPath ($base+'.lua') -Value 'return ''lua-fixture-ok''' -Encoding utf8; Set-Content -LiteralPath ($base+'_rename_src.txt') -Value 'rename-source' -Encoding utf8; Set-Content -LiteralPath ($base+'_loadlib_stub.dll') -Value 'not-a-dll' -Encoding ascii;" +if errorlevel 1 exit /b 1 +> "%DIR%\setup_done.txt" echo ok + +exit /b 0 diff --git a/tests/test_utf8_wrappers.lua b/tests/test_utf8_wrappers.lua new file mode 100644 index 0000000..9536c92 --- /dev/null +++ b/tests/test_utf8_wrappers.lua @@ -0,0 +1,210 @@ +local PASS, FAIL = 0, 0 +-- This test intentionally targets Windows path APIs and wrappers only. + +local function test(name, fn) + io.write("[TEST] " .. name) + io.flush() + local ok, err = pcall(fn) + if ok then + io.write("\r[PASS] " .. name .. "\n") + PASS = PASS + 1 + else + io.write("\r[FAIL] " .. name .. ": " .. tostring(err) .. "\n") + FAIL = FAIL + 1 + end +end + +local DIR_WINDOWS = "tests\\_unicode_fixture" +-- Keep in sync with codepoint-generated name in tests/test_setup.cmd. +local NAME = "Ελλ_中文_한국_عربي_кирил_देवनागरी" +local BASE +local PATH_RENAME_SRC +local PATH_RENAMED +local PATH_WRITE +local PATH_EXEC_MARKER +local PATH_LOADLIB_STUB + +local function file_exists(path) + local f = io.open(path, "rb") + if f then + f:close() + return true + end + return false +end + +local function read_all(path) + local f, err = io.open(path, "rb") + assert(f, err) + local data = f:read("*a") + f:close() + return data +end + +local function has_non_ascii(s) + for i = 1, #s do + if s:byte(i) > 127 then + return true + end + end + return false +end + +local function command_ok(ret) + -- LuaJIT/Lua 5.1 on Windows may return boolean true, while Lua 5.2+ + -- style semantics can return numeric zero for success. Failure values differ + -- by version, so callers should only use this as a success predicate. + return ret == true or ret == 0 +end + +local function cmd_quote(path) + -- Escape CMD metacharacters with '^' and double embedded quotes. + -- In the Lua pattern below, leading '%%' matches literal '%', while + -- '%X' escapes special pattern characters used in the class. + local escaped = path:gsub('[%%%^&|<>()!%[%]]', '^%1'):gsub('"', '""') + return '"' .. escaped .. '"' +end + +local function cleanup_fixtures() + os.execute('cmd /c rmdir /s /q ' .. cmd_quote(DIR_WINDOWS)) +end + +local function ensure_fixture(path, content) + if file_exists(path) then + return + end + local f = assert(io.open(path, "wb")) + local success, err = f:write(content) + if not success then + f:close() + error(err) + end + f:close() +end + +local function run_suite() + local setup_ret = os.execute("cmd /c tests\\test_setup.cmd") + assert(command_ok(setup_ret), "test_setup.cmd failed: " .. tostring(setup_ret)) + assert(file_exists(DIR_WINDOWS .. "\\setup_done.txt"), "setup_done.txt is missing") + assert(has_non_ascii(NAME), "fixture name is not Unicode: " .. tostring(NAME)) + BASE = DIR_WINDOWS .. "\\" .. NAME + PATH_RENAME_SRC = BASE .. "_rename_src.txt" + PATH_RENAMED = BASE .. "_renamed.txt" + PATH_WRITE = BASE .. "_write.txt" + PATH_EXEC_MARKER = BASE .. "_exec_marker.txt" + PATH_LOADLIB_STUB = BASE .. "_loadlib_stub.dll" + + ensure_fixture(BASE .. ".txt", "fixture-content") + ensure_fixture(BASE .. ".lua", "return 'lua-fixture-ok'") + ensure_fixture(PATH_RENAME_SRC, "rename-source") + ensure_fixture(PATH_LOADLIB_STUB, "not-a-dll") + + test("io.open read trusted Unicode fixture", function() + local data = read_all(BASE .. ".txt") + assert(data:find("fixture%-content", 1, false), "unexpected fixture data") + end) + + test("loadfile Unicode path", function() + local chunk, err = loadfile(BASE .. ".lua") + assert(chunk, err) + assert(chunk() == "lua-fixture-ok") + end) + + test("dofile Unicode path", function() + assert(dofile(BASE .. ".lua") == "lua-fixture-ok") + end) + + test("require Lua module from Unicode path", function() + local old_path = package.path + package.path = DIR_WINDOWS .. "\\?.lua;" .. old_path + package.loaded[NAME] = nil + local ok, mod = pcall(require, NAME) + package.path = old_path + package.loaded[NAME] = nil + assert(ok, tostring(mod)) + assert(mod == "lua-fixture-ok") + end) + + test("io.open write Unicode path", function() + assert(has_non_ascii(PATH_WRITE), "path must contain Unicode") + local f, err = io.open(PATH_WRITE, "wb") + assert(f, err) + local old_output = io.output() + io.output(f) + io.write("written-by-luajit") + io.output(old_output) + f:close() + assert(file_exists(PATH_WRITE), "written file is missing") + local data = read_all(PATH_WRITE) + assert(data:find("written%-by%-luajit", 1, false), "write verification failed") + end) + + test("os.rename Unicode -> Unicode", function() + assert(file_exists(PATH_RENAME_SRC), "precondition failed: source is missing") + os.remove(PATH_RENAMED) + local ok, err = os.rename(PATH_RENAME_SRC, PATH_RENAMED) + assert(ok, err) + assert(file_exists(PATH_RENAMED), "destination file is missing") + assert(not file_exists(PATH_RENAME_SRC), "source file still exists") + end) + + test("os.remove Unicode", function() + assert(has_non_ascii(PATH_WRITE), "path must contain Unicode") + assert(file_exists(PATH_WRITE), "precondition failed: writable file is missing") + local ok, err = os.remove(PATH_WRITE) + assert(ok, err) + assert(not file_exists(PATH_WRITE), "file still exists") + end) + + test("io.popen Unicode command arg", function() + local target = cmd_quote(PATH_RENAMED) + local cmd = 'cmd /c if exist ' .. target .. ' (echo 1) else (echo 0)' + local f, err = io.popen(cmd, "r") + assert(f, err) + local out = f:read("*l") + f:close() + assert(out == "1", "io.popen command did not see Unicode path") + end) + + test("os.execute Unicode command arg", function() + assert(has_non_ascii(PATH_EXEC_MARKER), "path must contain Unicode") + os.remove(PATH_EXEC_MARKER) + local ret = os.execute('cmd /c type nul > ' .. cmd_quote(PATH_EXEC_MARKER)) + assert(command_ok(ret), "os.execute returned " .. tostring(ret)) + assert(file_exists(PATH_EXEC_MARKER), "marker file was not created") + os.remove(PATH_EXEC_MARKER) + end) + + test("os.getenv Unicode value", function() + local val = os.getenv("IAT_TEST_VAR") + assert(type(val) == "string", "unexpected env value type: expected string") + assert(val ~= "", "unexpected empty env value") + if not has_non_ascii(val) then + io.stderr:write("[WARN] IAT_TEST_VAR lost Unicode in this shell environment\n") + else + assert(val == NAME, "unexpected Unicode env value: " .. tostring(val)) + end + end) + + test("package.loadlib Unicode C path", function() + assert(file_exists(PATH_LOADLIB_STUB), "loadlib stub fixture is missing") + local loader, err = package.loadlib(PATH_LOADLIB_STUB, "luaopen_jit") + assert(not loader, "stub load unexpectedly succeeded") + local msg = tostring(err) + local msg_l = msg:lower() + assert(not msg_l:find("no such file", 1, true), "unexpected file-not-found error: " .. msg) + assert(not msg_l:find("could not be found", 1, true), "unexpected missing-file error: " .. msg) + end) +end + +local ok, err = xpcall(run_suite, debug.traceback) +cleanup_fixtures() +if not ok then + io.write("\n[FATAL] " .. tostring(err) .. "\n") + os.exit(1) +end + +io.write(string.format("\n%d passed, %d failed\n", PASS, FAIL)) +if FAIL > 0 then + os.exit(1) +end