perf: optimize native binary size and startup time

build.mill

@@ -271,7 +271,6 @@ object sjsonnet extends VersionFileModule {
 
     def mvnDeps = super.mvnDeps() ++
       Seq(
-        mvn"com.github.lolgab::scala-native-crypto::0.3.0",
         mvn"org.virtuslab::scala-yaml::0.3.1"
       )
 
@@ -327,6 +326,14 @@ object sjsonnet extends VersionFileModule {
       )
     }
 
+    override def nativeLink = Task {
+      val linked = super.nativeLink()
+      val stripped = Task.ctx().dest / "out"
+      os.copy(linked.path, stripped)
+      os.proc("strip", stripped.toString).call()
+      PathRef(stripped)
+    }
+
     object test extends ScalaNativeTests with CrossTests {
       def releaseMode = ReleaseMode.Debug
       def nativeMultithreading = None

sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala

@@ -85,7 +85,7 @@ object SjsonnetMainBase {
       wd: os.Path,
       allowedInputs: Option[Set[os.Path]],
       importer: Option[Importer],
-      std: Val.Obj): Int =
+      std: => Val.Obj): Int =
     main0(
       args,
       parseCache,
@@ -108,7 +108,7 @@ object SjsonnetMainBase {
       wd: os.Path,
       allowedInputs: Option[Set[os.Path]] = None,
       importer: Option[Importer] = None,
-      std: Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module,
+      std: => Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module,
       jsonnetPathEnv: Option[String] = None): Int =
     main0(
       args,
@@ -133,7 +133,7 @@ object SjsonnetMainBase {
       wd: os.Path,
       allowedInputs: Option[Set[os.Path]],
       importer: Option[Importer],
-      std: Val.Obj,
+      std: => Val.Obj,
       jsonnetPathEnv: Option[String],
       rawOutputStream: OutputStream): Int = {
 
@@ -390,7 +390,7 @@ object SjsonnetMainBase {
       wd: os.Path,
       importer: Importer,
       warnLogger: Evaluator.Logger,
-      std: Val.Obj,
+      std: => Val.Obj,
       evaluatorOverride: Option[Evaluator] = None,
       debugStats: DebugStats = null,
       profileOpt: Option[String] = None,
@@ -445,7 +445,7 @@ object SjsonnetMainBase {
       settings = settings,
       storePos = (position: Position) => if (config.yamlDebug.value) currentPos = position else (),
       logger = warnLogger,
-      std = std,
+      stdParam = std,
       variableResolver = _ => None,
       debugStats = debugStats,
       formatCache = FormatCache.SharedDefault

sjsonnet/src-native/sjsonnet/Platform.scala

@@ -6,6 +6,9 @@ import java.util
 import java.util.Base64
 import java.util.zip.GZIPOutputStream
 import scala.scalanative.regex.Pattern
+import scala.scalanative.unsafe._
+import scala.scalanative.unsigned._
+import scala.scalanative.posix.dlfcn._
 import scala.annotation.nowarn
 import scala.collection.mutable
 import org.virtuslab.yaml.*
@@ -186,8 +189,121 @@ object Platform {
     new String(out)
   }
 
-  private def computeHash(algorithm: String, s: String): String =
-    bytesToHex(java.security.MessageDigest.getInstance(algorithm).digest(s.getBytes(UTF_8)))
+  // --- OpenSSL lazy loading via dlopen (avoids libcrypto startup cost) ---
+
+  private type EVP_MD_Ptr = CVoidPtr
+  private type EVP_MD_CTX_Ptr = CVoidPtr
+
+  private type EVP_get_digestbyname_t = CFuncPtr1[CString, EVP_MD_Ptr]
+  private type EVP_MD_CTX_new_t = CFuncPtr0[EVP_MD_CTX_Ptr]
+  private type EVP_MD_CTX_free_t = CFuncPtr1[EVP_MD_CTX_Ptr, Unit]
+  private type EVP_DigestInit_t = CFuncPtr2[EVP_MD_CTX_Ptr, EVP_MD_Ptr, CInt]
+  private type EVP_DigestUpdate_t = CFuncPtr3[EVP_MD_CTX_Ptr, Ptr[Byte], CSize, CInt]
+  private type EVP_DigestFinal_t = CFuncPtr3[EVP_MD_CTX_Ptr, Ptr[Byte], Ptr[CUnsignedInt], CInt]
+
+  private lazy val cryptoHandle: CVoidPtr = {
+    def tryOpen(names: List[String])(implicit z: Zone): CVoidPtr = names match {
+      case Nil          => null
+      case name :: rest =>
+        val h = dlopen(toCString(name), RTLD_LAZY | RTLD_LOCAL)
+        if (h != null) h else tryOpen(rest)
+    }
+    Zone.acquire { implicit z =>
+      tryOpen(
+        List(
+          "/opt/homebrew/lib/libcrypto.dylib",
+          "/opt/homebrew/lib/libcrypto.3.dylib",
+          "/usr/local/lib/libcrypto.dylib",
+          "/usr/local/lib/libcrypto.3.dylib",
+          "libcrypto.dylib",
+          "libcrypto.3.dylib",
+          "libcrypto.1.1.dylib",
+          "/usr/lib/x86_64-linux-gnu/libcrypto.so",
+          "/usr/lib/aarch64-linux-gnu/libcrypto.so",
+          "libcrypto.so",
+          "libcrypto.so.3",
+          "libcrypto.so.1.1"
+        )
+      )
+    }
+  }
+
+  // Cache function pointers to avoid repeated dlsym lookups on every hash computation
+  private lazy val cryptoFuncs: Option[
+    (
+        EVP_get_digestbyname_t,
+        EVP_MD_CTX_new_t,
+        EVP_MD_CTX_free_t,
+        EVP_DigestInit_t,
+        EVP_DigestUpdate_t,
+        EVP_DigestFinal_t
+    )
+  ] = {
+    if (cryptoHandle == null) None
+    else
+      Zone.acquire { implicit z =>
+        def loadSym(name: String): CVoidPtr = {
+          val ptr = dlsym(cryptoHandle, toCString(name))
+          if (ptr == null) throw new RuntimeException(s"OpenSSL symbol not found: $name")
+          ptr
+        }
+        Some(
+          (
+            CFuncPtr.fromPtr[EVP_get_digestbyname_t](loadSym("EVP_get_digestbyname")),
+            CFuncPtr.fromPtr[EVP_MD_CTX_new_t](loadSym("EVP_MD_CTX_new")),
+            CFuncPtr.fromPtr[EVP_MD_CTX_free_t](loadSym("EVP_MD_CTX_free")),
+            CFuncPtr.fromPtr[EVP_DigestInit_t](loadSym("EVP_DigestInit")),
+            CFuncPtr.fromPtr[EVP_DigestUpdate_t](loadSym("EVP_DigestUpdate")),
+            CFuncPtr.fromPtr[EVP_DigestFinal_t](loadSym("EVP_DigestFinal"))
+          )
+        )
+      }
+  }
+
+  private def computeHash(algorithm: String, s: String): String = {
+    val funcs = cryptoFuncs.getOrElse(
+      throw new RuntimeException(s"libcrypto not found; install OpenSSL to use $algorithm")
+    )
+    val (evpGetDigest, ctxNew, ctxFree, digestInit, digestUpdate, digestFinal) = funcs
+
+    Zone.acquire { implicit z =>
+      val evpName = algorithm match {
+        case "SHA-1"   => "SHA1"
+        case "SHA-256" => "SHA256"
+        case "SHA-512" => "SHA512"
+        case other     => other
+      }
+      val md = evpGetDigest(toCString(evpName))
+      if (md == null)
+        throw new RuntimeException(s"Hash algorithm not found in OpenSSL: $algorithm")
+
+      val ctx = ctxNew()
+      if (ctx == null)
+        throw new RuntimeException("EVP_MD_CTX_new failed")
+
+      try {
+        val data = s.getBytes(UTF_8)
+        val dataPtr = if (data.isEmpty) alloc[Byte](1) else data.at(0)
+        val hashBuf = alloc[Byte](64)
+        val hashLen = alloc[CUnsignedInt](1)
+
+        if (
+          digestInit(ctx, md) != 1 ||
+          digestUpdate(ctx, dataPtr, data.length.toUSize) != 1 ||
+          digestFinal(ctx, hashBuf, hashLen) != 1
+        )
+          throw new RuntimeException(s"Hash computation failed for $algorithm")
+
+        val len = (!hashLen).toInt
+        val result = new Array[Byte](len)
+        var i = 0
+        while (i < len) { result(i) = !(hashBuf + i); i += 1 }
+        bytesToHex(result)
+      } finally {
+        ctxFree(ctx)
+      }
+    }
+  }
 
   def md5(s: String): String = computeHash("MD5", s)
 

sjsonnet/src-native/sjsonnet/SjsonnetMain.scala

@@ -5,6 +5,9 @@ import scala.scalanative.libc.stdio
 
 object SjsonnetMain {
   def main(args: Array[String]): Unit = {
+    val stdLib = new sjsonnet.stdlib.StdLibModule(nativeFunctions =
+      Map.from(NativeGzip.functions ++ NativeRegex.functions)
+    )
     val exitCode = SjsonnetMainBase.main0(
       args,
       new DefaultParseCache,
@@ -14,9 +17,7 @@ object SjsonnetMain {
       os.pwd,
       None,
       None,
-      new sjsonnet.stdlib.StdLibModule(nativeFunctions =
-        Map.from(NativeGzip.functions ++ NativeRegex.functions)
-      ).module,
+      stdLib.module,
       None,
       new NativeOutputStream(stdio.stdout)
     )

sjsonnet/src/sjsonnet/Format.scala

@@ -309,7 +309,7 @@ object Format {
    * large format strings (e.g. 605KB large_string_template with 256 interpolations), this avoids
    * the overhead of running parser combinators over hundreds of KB of literal text.
    */
-  private def scanFormat(s: String): RuntimeFormat = {
+  private def scanFormat(s: String, sourceAsciiSafe: Boolean = false): RuntimeFormat = {
     val len = s.length
     val specsBuilder = new scala.collection.mutable.ArrayBuilder.ofLong
     var labelsBuilder: java.util.ArrayList[String] = null
@@ -329,7 +329,7 @@ object Format {
     val leadingStart = 0
     val leadingEnd = if (pos < 0) len else pos
     staticChars += leadingEnd - leadingStart
-    if (allLiteralsAscii && !isAsciiJsonSafeRange(s, leadingStart, leadingEnd))
+    if (!sourceAsciiSafe && allLiteralsAscii && !isAsciiJsonSafeRange(s, leadingStart, leadingEnd))
       allLiteralsAscii = false
 
     while (pos >= 0 && pos < len) {
@@ -470,7 +470,7 @@ object Format {
       litStartsBuilder += litStart
       litEndsBuilder += litEnd
       staticChars += litEnd - litStart
-      if (allLiteralsAscii && !isAsciiJsonSafeRange(s, litStart, litEnd))
+      if (!sourceAsciiSafe && allLiteralsAscii && !isAsciiJsonSafeRange(s, litStart, litEnd))
         allLiteralsAscii = false
 
       pos = nextPct
@@ -1131,11 +1131,12 @@ object Format {
    * instance, bypassing [[FormatCache]] since each literal format string is unique and already
    * cached within the AST.
    */
-  class PartialApplyFmt(fmt: String) extends Val.Builtin1("format", "values") {
+  class PartialApplyFmt(fmt: String, sourceAsciiSafe: Boolean = false)
+      extends Val.Builtin1("format", "values") {
     // Pre-parse the format string at construction time (during static optimization).
     // Uses the hand-written scanner instead of fastparse for faster parsing of large format strings.
     // Each PartialApplyFmt instance caches its own parsed format, so no external cache needed.
-    private val parsed = scanFormat(fmt)
+    private val parsed = scanFormat(fmt, sourceAsciiSafe)
     def evalRhs(values0: Eval, ev: EvalScope, pos: Position): Val =
       format(parsed, values0.value, pos)(ev)
   }

sjsonnet/src/sjsonnet/Interpreter.scala

@@ -43,12 +43,14 @@ class Interpreter(
     settings: Settings,
     storePos: Position => Unit,
     logger: Evaluator.Logger,
-    std: Val.Obj,
+    stdParam: => Val.Obj,
     variableResolver: String => Option[Expr],
     val debugStats: DebugStats,
     formatCache: FormatCache
 ) { self =>
 
+  lazy val std: Val.Obj = stdParam
+
   def this(
       extVars: Map[String, String],
       tlaVars: Map[String, String],
@@ -58,7 +60,7 @@ class Interpreter(
       settings: Settings = Settings.default,
       storePos: Position => Unit = null,
       logger: (Boolean, String) => Unit = null,
-      std: Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module,
+      std: => Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module,
       variableResolver: String => Option[Expr] = _ => None) =
     this(
       key => extVars.get(key).map(ExternalVariable.code),

sjsonnet/src/sjsonnet/StaticOptimizer.scala

@@ -24,13 +24,14 @@ import ScopedExprTransform.*
 class StaticOptimizer(
     ev: EvalScope,
     variableResolver: String => Option[Expr],
-    std: Val.Obj,
+    stdParam: => Val.Obj,
     internedStrings: mutable.HashMap[String, String],
     internedStaticFieldSets: mutable.HashMap[
       Val.StaticObjectFieldSet,
       java.util.LinkedHashMap[String, java.lang.Boolean]
     ])
     extends ScopedExprTransform {
+  lazy val std: Val.Obj = stdParam
   def optimize(e: Expr): Expr = transform(e)
 
   override def transform(_e: Expr): Expr = {
@@ -59,16 +60,22 @@ class StaticOptimizer(
         InSuper(pos, lhs, selfIdx)
       case b2 @ BinaryOp(pos, lhs: Val.Str, BinaryOp.OP_%, rhs) =>
         try {
+          val asciiSafe = lhs.isInstanceOf[Val.AsciiSafeStr]
           rhs match {
             case r: Val =>
-              val partial = new Format.PartialApplyFmt(lhs.str)
+              val partial = new Format.PartialApplyFmt(lhs.str, asciiSafe)
               try partial.evalRhs(r, ev, pos).asInstanceOf[Expr]
               catch {
                 case _: Exception =>
                   ApplyBuiltin1(pos, partial, rhs, tailstrict = false)
               }
             case _ =>
-              ApplyBuiltin1(pos, new Format.PartialApplyFmt(lhs.str), rhs, tailstrict = false)
+              ApplyBuiltin1(
+                pos,
+                new Format.PartialApplyFmt(lhs.str, asciiSafe),
+                rhs,
+                tailstrict = false
+              )
           }
         } catch { case _: Exception => b2 }
 

sjsonnet/src/sjsonnet/Val.scala

@@ -255,10 +255,10 @@ object Val {
    * Shared singletons for runtime boolean results — avoids per-comparison allocation. WARNING:
    * These singletons have mutable `var pos` (inherited from Expr). Their `pos` must NEVER be
    * mutated. The evaluation model is single-threaded, but mutating shared singleton state would
-   * corrupt all subsequent uses.
+   * corrupt all subsequent uses. (Lazy to defer startup cost)
    */
-  val staticTrue: Bool = True(new Position(null, -1))
-  val staticFalse: Bool = False(new Position(null, -1))
+  lazy val staticTrue: Bool = True(new Position(null, -1))
+  lazy val staticFalse: Bool = False(new Position(null, -1))
 
   /**
    * Returns a shared singleton boolean. Use for runtime comparison results where position is not
@@ -271,10 +271,10 @@ object Val {
   /**
    * Pre-allocated pool of Val.Num for small non-negative integers 0–255. Used by Evaluator
    * arithmetic fast paths to avoid per-operation allocation. Position is synthetic — acceptable for
-   * intermediate runtime results.
+   * intermediate runtime results. (Lazy to defer startup cost)
    */
   private val numCacheSize = 256
-  private val numCache: Array[Num] = {
+  private lazy val numCache: Array[Num] = {
     val pos = new Position(null, -1)
     val arr = new Array[Num](numCacheSize)
     var i = 0
@@ -317,9 +317,9 @@ object Val {
 
   /**
    * Singleton null for runtime results where position is not meaningful. Safe in single-threaded
-   * evaluation. See staticTrue/staticFalse for rationale.
+   * evaluation. See staticTrue/staticFalse for rationale. (Lazy to defer startup cost)
    */
-  val staticNull: Val.Null = Val.Null(new Position(null, -1))
+  lazy val staticNull: Val.Null = Val.Null(new Position(null, -1))
 
   /**
    * Rope string: O(1) concatenation via inline tree nodes.