diff --git a/build.mill b/build.mill index 2c56351a..d151baa2 100644 --- a/build.mill +++ b/build.mill @@ -271,7 +271,6 @@ object sjsonnet extends VersionFileModule { def mvnDeps = super.mvnDeps() ++ Seq( - mvn"com.github.lolgab::scala-native-crypto::0.3.0", mvn"org.virtuslab::scala-yaml::0.3.1" ) @@ -327,6 +326,14 @@ object sjsonnet extends VersionFileModule { ) } + override def nativeLink = Task { + val linked = super.nativeLink() + val stripped = Task.ctx().dest / "out" + os.copy(linked.path, stripped) + os.proc("strip", stripped.toString).call() + PathRef(stripped) + } + object test extends ScalaNativeTests with CrossTests { def releaseMode = ReleaseMode.Debug def nativeMultithreading = None diff --git a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala index d5c1641d..2d96f9f5 100644 --- a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala +++ b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala @@ -85,7 +85,7 @@ object SjsonnetMainBase { wd: os.Path, allowedInputs: Option[Set[os.Path]], importer: Option[Importer], - std: Val.Obj): Int = + std: => Val.Obj): Int = main0( args, parseCache, @@ -108,7 +108,7 @@ object SjsonnetMainBase { wd: os.Path, allowedInputs: Option[Set[os.Path]] = None, importer: Option[Importer] = None, - std: Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module, + std: => Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module, jsonnetPathEnv: Option[String] = None): Int = main0( args, @@ -133,7 +133,7 @@ object SjsonnetMainBase { wd: os.Path, allowedInputs: Option[Set[os.Path]], importer: Option[Importer], - std: Val.Obj, + std: => Val.Obj, jsonnetPathEnv: Option[String], rawOutputStream: OutputStream): Int = { @@ -390,7 +390,7 @@ object SjsonnetMainBase { wd: os.Path, importer: Importer, warnLogger: Evaluator.Logger, - std: Val.Obj, + std: => Val.Obj, evaluatorOverride: Option[Evaluator] = None, debugStats: DebugStats = null, profileOpt: Option[String] = None, @@ -445,7 +445,7 @@ object SjsonnetMainBase { settings = settings, storePos = (position: Position) => if (config.yamlDebug.value) currentPos = position else (), logger = warnLogger, - std = std, + stdParam = std, variableResolver = _ => None, debugStats = debugStats, formatCache = FormatCache.SharedDefault diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index ab8f580e..13341c4a 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -6,6 +6,9 @@ import java.util import java.util.Base64 import java.util.zip.GZIPOutputStream import scala.scalanative.regex.Pattern +import scala.scalanative.unsafe._ +import scala.scalanative.unsigned._ +import scala.scalanative.posix.dlfcn._ import scala.annotation.nowarn import scala.collection.mutable import org.virtuslab.yaml.* @@ -186,8 +189,121 @@ object Platform { new String(out) } - private def computeHash(algorithm: String, s: String): String = - bytesToHex(java.security.MessageDigest.getInstance(algorithm).digest(s.getBytes(UTF_8))) + // --- OpenSSL lazy loading via dlopen (avoids libcrypto startup cost) --- + + private type EVP_MD_Ptr = CVoidPtr + private type EVP_MD_CTX_Ptr = CVoidPtr + + private type EVP_get_digestbyname_t = CFuncPtr1[CString, EVP_MD_Ptr] + private type EVP_MD_CTX_new_t = CFuncPtr0[EVP_MD_CTX_Ptr] + private type EVP_MD_CTX_free_t = CFuncPtr1[EVP_MD_CTX_Ptr, Unit] + private type EVP_DigestInit_t = CFuncPtr2[EVP_MD_CTX_Ptr, EVP_MD_Ptr, CInt] + private type EVP_DigestUpdate_t = CFuncPtr3[EVP_MD_CTX_Ptr, Ptr[Byte], CSize, CInt] + private type EVP_DigestFinal_t = CFuncPtr3[EVP_MD_CTX_Ptr, Ptr[Byte], Ptr[CUnsignedInt], CInt] + + private lazy val cryptoHandle: CVoidPtr = { + def tryOpen(names: List[String])(implicit z: Zone): CVoidPtr = names match { + case Nil => null + case name :: rest => + val h = dlopen(toCString(name), RTLD_LAZY | RTLD_LOCAL) + if (h != null) h else tryOpen(rest) + } + Zone.acquire { implicit z => + tryOpen( + List( + "/opt/homebrew/lib/libcrypto.dylib", + "/opt/homebrew/lib/libcrypto.3.dylib", + "/usr/local/lib/libcrypto.dylib", + "/usr/local/lib/libcrypto.3.dylib", + "libcrypto.dylib", + "libcrypto.3.dylib", + "libcrypto.1.1.dylib", + "/usr/lib/x86_64-linux-gnu/libcrypto.so", + "/usr/lib/aarch64-linux-gnu/libcrypto.so", + "libcrypto.so", + "libcrypto.so.3", + "libcrypto.so.1.1" + ) + ) + } + } + + // Cache function pointers to avoid repeated dlsym lookups on every hash computation + private lazy val cryptoFuncs: Option[ + ( + EVP_get_digestbyname_t, + EVP_MD_CTX_new_t, + EVP_MD_CTX_free_t, + EVP_DigestInit_t, + EVP_DigestUpdate_t, + EVP_DigestFinal_t + ) + ] = { + if (cryptoHandle == null) None + else + Zone.acquire { implicit z => + def loadSym(name: String): CVoidPtr = { + val ptr = dlsym(cryptoHandle, toCString(name)) + if (ptr == null) throw new RuntimeException(s"OpenSSL symbol not found: $name") + ptr + } + Some( + ( + CFuncPtr.fromPtr[EVP_get_digestbyname_t](loadSym("EVP_get_digestbyname")), + CFuncPtr.fromPtr[EVP_MD_CTX_new_t](loadSym("EVP_MD_CTX_new")), + CFuncPtr.fromPtr[EVP_MD_CTX_free_t](loadSym("EVP_MD_CTX_free")), + CFuncPtr.fromPtr[EVP_DigestInit_t](loadSym("EVP_DigestInit")), + CFuncPtr.fromPtr[EVP_DigestUpdate_t](loadSym("EVP_DigestUpdate")), + CFuncPtr.fromPtr[EVP_DigestFinal_t](loadSym("EVP_DigestFinal")) + ) + ) + } + } + + private def computeHash(algorithm: String, s: String): String = { + val funcs = cryptoFuncs.getOrElse( + throw new RuntimeException(s"libcrypto not found; install OpenSSL to use $algorithm") + ) + val (evpGetDigest, ctxNew, ctxFree, digestInit, digestUpdate, digestFinal) = funcs + + Zone.acquire { implicit z => + val evpName = algorithm match { + case "SHA-1" => "SHA1" + case "SHA-256" => "SHA256" + case "SHA-512" => "SHA512" + case other => other + } + val md = evpGetDigest(toCString(evpName)) + if (md == null) + throw new RuntimeException(s"Hash algorithm not found in OpenSSL: $algorithm") + + val ctx = ctxNew() + if (ctx == null) + throw new RuntimeException("EVP_MD_CTX_new failed") + + try { + val data = s.getBytes(UTF_8) + val dataPtr = if (data.isEmpty) alloc[Byte](1) else data.at(0) + val hashBuf = alloc[Byte](64) + val hashLen = alloc[CUnsignedInt](1) + + if ( + digestInit(ctx, md) != 1 || + digestUpdate(ctx, dataPtr, data.length.toUSize) != 1 || + digestFinal(ctx, hashBuf, hashLen) != 1 + ) + throw new RuntimeException(s"Hash computation failed for $algorithm") + + val len = (!hashLen).toInt + val result = new Array[Byte](len) + var i = 0 + while (i < len) { result(i) = !(hashBuf + i); i += 1 } + bytesToHex(result) + } finally { + ctxFree(ctx) + } + } + } def md5(s: String): String = computeHash("MD5", s) diff --git a/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala index cb2deb55..a2c4b74f 100644 --- a/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala @@ -5,6 +5,9 @@ import scala.scalanative.libc.stdio object SjsonnetMain { def main(args: Array[String]): Unit = { + val stdLib = new sjsonnet.stdlib.StdLibModule(nativeFunctions = + Map.from(NativeGzip.functions ++ NativeRegex.functions) + ) val exitCode = SjsonnetMainBase.main0( args, new DefaultParseCache, @@ -14,9 +17,7 @@ object SjsonnetMain { os.pwd, None, None, - new sjsonnet.stdlib.StdLibModule(nativeFunctions = - Map.from(NativeGzip.functions ++ NativeRegex.functions) - ).module, + stdLib.module, None, new NativeOutputStream(stdio.stdout) ) diff --git a/sjsonnet/src/sjsonnet/Format.scala b/sjsonnet/src/sjsonnet/Format.scala index 731c341d..dbabc599 100644 --- a/sjsonnet/src/sjsonnet/Format.scala +++ b/sjsonnet/src/sjsonnet/Format.scala @@ -309,7 +309,7 @@ object Format { * large format strings (e.g. 605KB large_string_template with 256 interpolations), this avoids * the overhead of running parser combinators over hundreds of KB of literal text. */ - private def scanFormat(s: String): RuntimeFormat = { + private def scanFormat(s: String, sourceAsciiSafe: Boolean = false): RuntimeFormat = { val len = s.length val specsBuilder = new scala.collection.mutable.ArrayBuilder.ofLong var labelsBuilder: java.util.ArrayList[String] = null @@ -329,7 +329,7 @@ object Format { val leadingStart = 0 val leadingEnd = if (pos < 0) len else pos staticChars += leadingEnd - leadingStart - if (allLiteralsAscii && !isAsciiJsonSafeRange(s, leadingStart, leadingEnd)) + if (!sourceAsciiSafe && allLiteralsAscii && !isAsciiJsonSafeRange(s, leadingStart, leadingEnd)) allLiteralsAscii = false while (pos >= 0 && pos < len) { @@ -470,7 +470,7 @@ object Format { litStartsBuilder += litStart litEndsBuilder += litEnd staticChars += litEnd - litStart - if (allLiteralsAscii && !isAsciiJsonSafeRange(s, litStart, litEnd)) + if (!sourceAsciiSafe && allLiteralsAscii && !isAsciiJsonSafeRange(s, litStart, litEnd)) allLiteralsAscii = false pos = nextPct @@ -1131,11 +1131,12 @@ object Format { * instance, bypassing [[FormatCache]] since each literal format string is unique and already * cached within the AST. */ - class PartialApplyFmt(fmt: String) extends Val.Builtin1("format", "values") { + class PartialApplyFmt(fmt: String, sourceAsciiSafe: Boolean = false) + extends Val.Builtin1("format", "values") { // Pre-parse the format string at construction time (during static optimization). // Uses the hand-written scanner instead of fastparse for faster parsing of large format strings. // Each PartialApplyFmt instance caches its own parsed format, so no external cache needed. - private val parsed = scanFormat(fmt) + private val parsed = scanFormat(fmt, sourceAsciiSafe) def evalRhs(values0: Eval, ev: EvalScope, pos: Position): Val = format(parsed, values0.value, pos)(ev) } diff --git a/sjsonnet/src/sjsonnet/Interpreter.scala b/sjsonnet/src/sjsonnet/Interpreter.scala index d40adf2e..82c6a335 100644 --- a/sjsonnet/src/sjsonnet/Interpreter.scala +++ b/sjsonnet/src/sjsonnet/Interpreter.scala @@ -43,12 +43,14 @@ class Interpreter( settings: Settings, storePos: Position => Unit, logger: Evaluator.Logger, - std: Val.Obj, + stdParam: => Val.Obj, variableResolver: String => Option[Expr], val debugStats: DebugStats, formatCache: FormatCache ) { self => + lazy val std: Val.Obj = stdParam + def this( extVars: Map[String, String], tlaVars: Map[String, String], @@ -58,7 +60,7 @@ class Interpreter( settings: Settings = Settings.default, storePos: Position => Unit = null, logger: (Boolean, String) => Unit = null, - std: Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module, + std: => Val.Obj = sjsonnet.stdlib.StdLibModule.Default.module, variableResolver: String => Option[Expr] = _ => None) = this( key => extVars.get(key).map(ExternalVariable.code), diff --git a/sjsonnet/src/sjsonnet/StaticOptimizer.scala b/sjsonnet/src/sjsonnet/StaticOptimizer.scala index 39a8803b..87fc8ecf 100644 --- a/sjsonnet/src/sjsonnet/StaticOptimizer.scala +++ b/sjsonnet/src/sjsonnet/StaticOptimizer.scala @@ -24,13 +24,14 @@ import ScopedExprTransform.* class StaticOptimizer( ev: EvalScope, variableResolver: String => Option[Expr], - std: Val.Obj, + stdParam: => Val.Obj, internedStrings: mutable.HashMap[String, String], internedStaticFieldSets: mutable.HashMap[ Val.StaticObjectFieldSet, java.util.LinkedHashMap[String, java.lang.Boolean] ]) extends ScopedExprTransform { + lazy val std: Val.Obj = stdParam def optimize(e: Expr): Expr = transform(e) override def transform(_e: Expr): Expr = { @@ -59,16 +60,22 @@ class StaticOptimizer( InSuper(pos, lhs, selfIdx) case b2 @ BinaryOp(pos, lhs: Val.Str, BinaryOp.OP_%, rhs) => try { + val asciiSafe = lhs.isInstanceOf[Val.AsciiSafeStr] rhs match { case r: Val => - val partial = new Format.PartialApplyFmt(lhs.str) + val partial = new Format.PartialApplyFmt(lhs.str, asciiSafe) try partial.evalRhs(r, ev, pos).asInstanceOf[Expr] catch { case _: Exception => ApplyBuiltin1(pos, partial, rhs, tailstrict = false) } case _ => - ApplyBuiltin1(pos, new Format.PartialApplyFmt(lhs.str), rhs, tailstrict = false) + ApplyBuiltin1( + pos, + new Format.PartialApplyFmt(lhs.str, asciiSafe), + rhs, + tailstrict = false + ) } } catch { case _: Exception => b2 } diff --git a/sjsonnet/src/sjsonnet/Val.scala b/sjsonnet/src/sjsonnet/Val.scala index 65fbc557..229735d0 100644 --- a/sjsonnet/src/sjsonnet/Val.scala +++ b/sjsonnet/src/sjsonnet/Val.scala @@ -255,10 +255,10 @@ object Val { * Shared singletons for runtime boolean results — avoids per-comparison allocation. WARNING: * These singletons have mutable `var pos` (inherited from Expr). Their `pos` must NEVER be * mutated. The evaluation model is single-threaded, but mutating shared singleton state would - * corrupt all subsequent uses. + * corrupt all subsequent uses. (Lazy to defer startup cost) */ - val staticTrue: Bool = True(new Position(null, -1)) - val staticFalse: Bool = False(new Position(null, -1)) + lazy val staticTrue: Bool = True(new Position(null, -1)) + lazy val staticFalse: Bool = False(new Position(null, -1)) /** * Returns a shared singleton boolean. Use for runtime comparison results where position is not @@ -271,10 +271,10 @@ object Val { /** * Pre-allocated pool of Val.Num for small non-negative integers 0–255. Used by Evaluator * arithmetic fast paths to avoid per-operation allocation. Position is synthetic — acceptable for - * intermediate runtime results. + * intermediate runtime results. (Lazy to defer startup cost) */ private val numCacheSize = 256 - private val numCache: Array[Num] = { + private lazy val numCache: Array[Num] = { val pos = new Position(null, -1) val arr = new Array[Num](numCacheSize) var i = 0 @@ -317,9 +317,9 @@ object Val { /** * Singleton null for runtime results where position is not meaningful. Safe in single-threaded - * evaluation. See staticTrue/staticFalse for rationale. + * evaluation. See staticTrue/staticFalse for rationale. (Lazy to defer startup cost) */ - val staticNull: Val.Null = Val.Null(new Position(null, -1)) + lazy val staticNull: Val.Null = Val.Null(new Position(null, -1)) /** * Rope string: O(1) concatenation via inline tree nodes. diff --git a/sjsonnet/src/sjsonnet/stdlib/StdLibModule.scala b/sjsonnet/src/sjsonnet/stdlib/StdLibModule.scala index fe59fc69..5c4855cd 100644 --- a/sjsonnet/src/sjsonnet/stdlib/StdLibModule.scala +++ b/sjsonnet/src/sjsonnet/stdlib/StdLibModule.scala @@ -21,14 +21,14 @@ final class StdLibModule( nativeFunctions.getOrElse(name.value.asString, Val.Null(pos)) } - // All functions including native and additional functions - val functions: Map[String, Val.Func] = allModuleFunctions ++ + // All functions including native and additional functions (lazy to defer startup cost) + lazy val functions: Map[String, Val.Func] = allModuleFunctions ++ additionalStdFunctions + ("native" -> nativeFunction) + ("trace" -> traceFunction) + ("extVar" -> extVarFunction) - val module: Val.Obj = Val.Obj.mk( + lazy val module: Val.Obj = Val.Obj.mk( null, functions.size + additionalStdMembers.size, functions.view.map { case (k, v) => @@ -52,8 +52,8 @@ final class StdLibModule( * - [[https://jsonnet.org/ref/stdlib.html#math std.pi]] */ object StdLibModule { - // Combine all functions from all modules - private val allModuleFunctions: Map[String, Val.Func] = ( + // Combine all functions from all modules (lazy to defer startup cost) + private[stdlib] lazy val allModuleFunctions: Map[String, Val.Func] = ( ArrayModule.functions ++ StringModule.functions ++ ObjectModule.functions ++ @@ -133,7 +133,7 @@ object StdLibModule { ) /** - * The default standard library module instance + * The default standard library module instance (lazy to defer startup cost) */ - val Default = new StdLibModule() + lazy val Default = new StdLibModule() } diff --git a/sjsonnet/test/src-jvm/sjsonnet/Example.java b/sjsonnet/test/src-jvm/sjsonnet/Example.java index 6b21e8b3..6db034e8 100644 --- a/sjsonnet/test/src-jvm/sjsonnet/Example.java +++ b/sjsonnet/test/src-jvm/sjsonnet/Example.java @@ -1,6 +1,7 @@ package sjsonnet; import scala.collection.immutable.Map$; +import scala.Function0; public class Example { public void example(){ @@ -13,8 +14,13 @@ public void example(){ os.package$.MODULE$.pwd(), scala.None$.empty(), scala.None$.empty(), - new sjsonnet.stdlib.StdLibModule(Map$.MODULE$.empty(), Map$.MODULE$.empty()).module(), - scala.None$.empty() + new Function0() { + public Val.Obj apply() { + return new sjsonnet.stdlib.StdLibModule(Map$.MODULE$.empty(), Map$.MODULE$.empty()).module(); + } + }, + scala.None$.empty(), + null ); } }