diff --git a/README.md b/README.md index cd606a98..e7e10f41 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,12 @@ Jawk fully implements POSIX AWK, and adds support for the most commonly used gaw - Builtins, available by default through the built-in GNU Awk compatibility extension: `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, `patsplit()`, `strtonum()`, `systime()`, `mktime()`, `strftime()`, `bindtextdomain()`, `dcgettext()`, `dcngettext()`, and `PROCINFO["sorted_in"]`-controlled array traversal - Arrays of arrays (`a[i][j]`) and typed regexp literals (`@/re/`) +- Source inclusion with `@include`, namespaces with `@namespace` and `ns::name`, and indirect function calls such as `@functionName(args)` - `BEGINFILE` / `ENDFILE` special patterns, with the `ERRNO` and `ARGIND` special variables, so a script can hook into the command-line file processing loop and skip unreadable files without a fatal error - The `nextfile` statement - The `IGNORECASE`, `SYMTAB`, and `FUNCTAB` special variables -As in gawk, the gawk-specific syntax is not special in `--posix` mode. +The gawk-specific `@` syntax and arrays-of-arrays syntax are rejected in `--posix` mode. ## CLI Example diff --git a/src/main/java/io/jawk/Awk.java b/src/main/java/io/jawk/Awk.java index 30c31567..b509484d 100644 --- a/src/main/java/io/jawk/Awk.java +++ b/src/main/java/io/jawk/Awk.java @@ -496,7 +496,10 @@ protected final T compileProgram( lastAst = null; if (!scripts.isEmpty()) { // Parse all script sources into a single AST - AwkParser parser = new AwkParser(this.extensionFunctions, settings.isPosix()); + AwkParser parser = new AwkParser( + this.extensionFunctions, + settings.isPosix(), + isSourceIncludeAllowed()); AstNode ast = parser.parse(scripts); lastAst = ast; if (ast != null) { @@ -521,6 +524,15 @@ protected final T compileProgram( return tuples; } + /** + * Returns whether scripts compiled by this engine may use {@code @include}. + * + * @return {@code true} for the standard engine + */ + protected boolean isSourceIncludeAllowed() { + return true; + } + /** * Compile an expression to evaluate (not a full script). * diff --git a/src/main/java/io/jawk/Cli.java b/src/main/java/io/jawk/Cli.java index 206102b4..7399b92e 100644 --- a/src/main/java/io/jawk/Cli.java +++ b/src/main/java/io/jawk/Cli.java @@ -376,7 +376,9 @@ private static void checkParameterHasArgument(String[] args, int argIdx) { } } - private static final Pattern INITIAL_VAR_PATTERN = Pattern.compile("([_a-zA-Z][_0-9a-zA-Z]*)=(.*)"); + private static final Pattern INITIAL_VAR_PATTERN = Pattern + .compile( + "((?:[_a-zA-Z][_0-9a-zA-Z]*::)?[_a-zA-Z][_0-9a-zA-Z]*)=(.*)"); /** * Parses a variable assignment passed via -v and stores it in the @@ -392,6 +394,9 @@ private static void addVariable(AwkSettings settings, String keyValue) { "keyValue \"" + keyValue + "\" must be of the form \"name=value\""); } String name = m.group(1); + if (name.startsWith("awk::")) { + name = name.substring("awk::".length()); + } String valueString = m.group(2); settings.putVariable(name, valueString); } diff --git a/src/main/java/io/jawk/SandboxedAwk.java b/src/main/java/io/jawk/SandboxedAwk.java index a9a2c2b2..50f35cb5 100644 --- a/src/main/java/io/jawk/SandboxedAwk.java +++ b/src/main/java/io/jawk/SandboxedAwk.java @@ -91,6 +91,11 @@ public AwkExpression compileExpression(String expression, boolean disableOptimiz return compileExpression(expression, disableOptimizeParam, new SandboxedCompiledAwkExpression()); } + @Override + protected boolean isSourceIncludeAllowed() { + return false; + } + @Override public AVM createAvm() { return createAvm(getSettings()); diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index df969415..9d88610f 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -25,6 +25,8 @@ import java.io.Closeable; import java.io.IOException; import java.io.PrintStream; +import java.util.AbstractSet; +import java.util.Iterator; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashMap; @@ -63,6 +65,8 @@ import io.jawk.intermediate.Tuple.CountTuple; import io.jawk.intermediate.Tuple.DereferenceTuple; import io.jawk.intermediate.Tuple.ExtensionTuple; +import io.jawk.intermediate.Tuple.IndirectCallTuple; +import io.jawk.intermediate.Tuple.IndirectFunctionTarget; import io.jawk.intermediate.Tuple.InputFieldTuple; import io.jawk.intermediate.Tuple.LongTuple; import io.jawk.intermediate.Tuple.PushDoubleTuple; @@ -578,16 +582,6 @@ private void initExtensions() { /** Random number generator used for rand() */ private final BSDRandom randomNumberGenerator = new BSDRandom(1); - /** - * Last seed value used with {@code srand()}. - *

- * The default seed for {@code rand()} in One True Awk is {@code 1}, so - * we initialize {@code oldseed} with this value to mimic that - * behaviour. This ensures deterministic sequences until the user - * explicitly calls {@code srand()}. - */ - private int oldseed = 1; - /** * Address of the END blocks section, read from the compiled program; * {@code null} for expression streams. @@ -731,7 +725,6 @@ private void resetTransientRuntimeState(List runtimeArguments, Map map = toMap(pop()); + Object scalarValue = JRT.getAssocArrayValue(map, idx); + push(new IndirectArrayArgumentReference(map, idx, scalarValue)); + position.next(); + break; + } case DEREF_ARRAY: { // stack[0] = array index Object idx = pop(); // idx @@ -1742,24 +1756,11 @@ private void executeTuples(PositionTracker position) // use the time of day for the seed seed = JRT.timeSeed(); } else { - Object o = pop(); - if (o instanceof Double) { - seed = ((Double) o).intValue(); - } else if (o instanceof Long) { - seed = ((Long) o).intValue(); - } else if (o instanceof Integer) { - seed = ((Integer) o).intValue(); - } else { - try { - seed = Integer.parseInt(o.toString()); - } catch (NumberFormatException nfe) { - seed = 0; - } - } + seed = (int) JRT.toDouble(pop()); } + int previousSeed = randomNumberGenerator.getSeed(); randomNumberGenerator.setSeed(seed); - push(oldseed); - oldseed = seed; + push(previousSeed); position.next(); break; } @@ -2219,6 +2220,72 @@ private void executeTuples(PositionTracker position) // position.next(); break; } + case INDIRECT_CALL: { + IndirectCallTuple callTuple = (IndirectCallTuple) tuple; + Object[] actualArguments = popArguments(callTuple.getNumActualParams()); + String requestedName = jrt.toAwkString(pop()); + String qualifiedName = normalizeIndirectFunctionName(requestedName); + IndirectFunctionTarget target = callTuple.getUserFunctions().get(qualifiedName); + if (target != null) { + resolveIndirectArguments(actualArguments, target); + long formalCount = target.getNumFormalParams(); + if (actualArguments.length > formalCount) { + jrt + .printWarning( + "gawk: " + + callTuple.getSourceName() + + ":" + + callTuple.getSourceLine() + + ": warning: function `" + + qualifiedName + + "' called with more arguments than declared"); + } + if (profiling) { + activeProfilingFunctions.push(new ActiveFunction(qualifiedName, tupleStartNanos)); + } + runtimeStack.pushFrame(formalCount, position.currentIndex()); + int copiedArgumentCount = Math.min(actualArguments.length, (int) formalCount); + for (int i = 0; i < copiedArgumentCount; i++) { + runtimeStack.setVariable(i, actualArguments[i], false); + } + position.jump(target.getAddress()); + break; + } + + String awkName = requestedName.startsWith("awk::") ? + requestedName.substring("awk::".length()) : requestedName; + BuiltinFunction builtin = BuiltinFunction.of(awkName); + if (builtin != null) { + resolveIndirectArguments(actualArguments, builtin); + push(invokeIndirectBuiltin(builtin, actualArguments, position.lineNumber())); + position.next(); + break; + } + ExtensionFunction extensionFunction = callTuple.getExtensionFunctions().get(awkName); + if (extensionFunction != null) { + resolveIndirectArguments(actualArguments, extensionFunction); + if (profiling) { + activeProfilingFunctions.push(new ActiveFunction(awkName, tupleStartNanos)); + } + try { + push( + invokeExtension( + extensionFunction, + actualArguments, + position.lineNumber(), + true)); + } finally { + if (profiling) { + recordFunctionExit(System.nanoTime()); + } + } + position.next(); + break; + } + throw new AwkRuntimeException( + position.lineNumber(), + "function `" + qualifiedName + "' is not defined"); + } case FUNCTION: { // important for compilation, // not needed for interpretation @@ -2415,55 +2482,12 @@ private void executeTuples(PositionTracker position) ExtensionFunction function = extensionTuple.getFunction(); long numArgs = extensionTuple.getArgCount(); boolean isInitial = extensionTuple.isInitial(); - // let extensions report diagnostics at the call location - currentLineNumber = position.lineNumber(); - - Object[] args = new Object[(int) numArgs]; - for (int i = (int) numArgs - 1; i >= 0; i--) { - args[i] = pop(); - } - - String extensionClassName = function.getExtensionClassName(); - JawkExtension extension = extensionInstances.get(extensionClassName); - if (extension == null) { - throw new AwkRuntimeException( - position.lineNumber(), - "Extension instance for class '" + extensionClassName - + "' is not registered"); - } - if (!(extension instanceof AbstractExtension)) { - throw new AwkRuntimeException( - position.lineNumber(), - "Extension instance for class '" + extensionClassName - + "' does not extend " - + AbstractExtension.class.getName()); - } - - Object retval = function.invoke((AbstractExtension) extension, args); - - // block if necessary - // (convert retval into the return value - // from the block operation ...) - if (isInitial && retval != null && retval instanceof BlockObject) { - retval = new BlockManager().block((BlockObject) retval); - } - // (... and proceed) - - if (retval == null) { - retval = ""; - } else - if (!(retval instanceof Number - || - retval instanceof String - || - retval instanceof Map - || - retval instanceof BlockObject)) { - // all other extension results are converted - // to a string (via Object.toString()) - retval = retval.toString(); - } - push(retval); + push( + invokeExtension( + function, + popArguments(numArgs), + position.lineNumber(), + isInitial)); position.next(); break; @@ -2796,12 +2820,223 @@ private void execLength(CountTuple tuple) { push(jrt.jrtGetInputField(0).toString().length()); return; } - Object value = pop(); - if (value instanceof Map) { - push((long) ((Map) value).size()); - } else { - push(jrt.toAwkString(value).length()); + push(lengthOf(pop())); + } + + private Object lengthOf(Object value) { + return value instanceof Map ? + Long.valueOf(((Map) value).size()) : Integer.valueOf(jrt.toAwkString(value).length()); + } + + private String normalizeIndirectFunctionName(String functionName) { + return functionName.startsWith("awk::") ? functionName.substring("awk::".length()) : functionName; + } + + private void resolveIndirectArguments( + Object[] actualArgumentsParam, + IndirectFunctionTarget target) { + for (int index = 0; index < actualArgumentsParam.length; index++) { + actualArgumentsParam[index] = resolveIndirectArgument( + actualArgumentsParam[index], + target.isArrayParameter(index), + false); + } + } + + private void resolveIndirectArguments( + Object[] actualArgumentsParam, + BuiltinFunction builtin) { + for (int index = 0; index < actualArgumentsParam.length; index++) { + boolean arrayArgument = builtin == BuiltinFunction.SPLIT && index == 1; + actualArgumentsParam[index] = resolveIndirectArgument( + actualArgumentsParam[index], + arrayArgument, + false); + } + } + + private void resolveIndirectArguments( + Object[] actualArgumentsParam, + ExtensionFunction function) { + boolean[] arrayArguments = new boolean[actualArgumentsParam.length]; + boolean[] rawValueArguments = new boolean[actualArgumentsParam.length]; + for (int index : function.collectAssocArrayIndexes(actualArgumentsParam.length)) { + arrayArguments[index] = true; + } + for (int index : function.collectRawValueIndexes(actualArgumentsParam.length)) { + rawValueArguments[index] = true; + } + for (int index = 0; index < actualArgumentsParam.length; index++) { + actualArgumentsParam[index] = resolveIndirectArgument( + actualArgumentsParam[index], + arrayArguments[index], + rawValueArguments[index]); + } + } + + private Object resolveIndirectArgument( + Object argument, + boolean arrayArgument, + boolean rawValueArgument) { + if (!(argument instanceof IndirectArgumentReference)) { + if (!(argument instanceof IndirectArrayArgumentReference)) { + return argument; + } + IndirectArrayArgumentReference reference = (IndirectArrayArgumentReference) argument; + return arrayArgument ? + ensureArrayInArray(reference.map, reference.key) : reference.scalarValue; + } + IndirectArgumentReference reference = (IndirectArgumentReference) argument; + if (!arrayArgument) { + return reference.scalarValue == null && !rawValueArgument ? BLANK : reference.scalarValue; + } + Object value = runtimeStack.getVariable(reference.offset, reference.global); + if (value == null) { + value = newAwkArray(); + runtimeStack.setVariable(reference.offset, value, reference.global); + } + return value; + } + + private Object invokeIndirectBuiltin( + BuiltinFunction builtin, + Object[] args, + int lineNumber) { + switch (builtin) { + case ATAN2: + requireIndirectArgumentCount(builtin, args, 2, 2, lineNumber); + return Math.atan2(JRT.toDouble(args[0]), JRT.toDouble(args[1])); + case CLOSE: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return jrt.jrtClose(jrt.toAwkString(args[0])); + case COS: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return Math.cos(JRT.toDouble(args[0])); + case EXP: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return Math.exp(JRT.toDouble(args[0])); + case GSUB: + case SUB: + requireIndirectArgumentCount(builtin, args, 2, 2, lineNumber); + return substituteInputLine( + builtin == BuiltinFunction.GSUB, + args[0], + args[1]); + case INDEX: + requireIndirectArgumentCount(builtin, args, 2, 2, lineNumber); + return jrt.index(jrt.toAwkString(args[0]), jrt.toAwkString(args[1])); + case INT: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return Long.valueOf((long) JRT.toDouble(args[0])); + case LENGTH: + requireIndirectArgumentCount(builtin, args, 0, 1, lineNumber); + return args.length == 0 ? Integer.valueOf(jrt.jrtGetInputField(0).toString().length()) : lengthOf(args[0]); + case LOG: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return Math.log(JRT.toDouble(args[0])); + case MATCH: + requireIndirectArgumentCount(builtin, args, 2, 2, lineNumber); + return jrt.matchPosition(jrt.toAwkString(args[0]), jrt.toAwkString(args[1])); + case RAND: + requireIndirectArgumentCount(builtin, args, 0, 0, lineNumber); + return randomNumberGenerator.nextDouble(); + case SIN: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return Math.sin(JRT.toDouble(args[0])); + case SPLIT: + requireIndirectArgumentCount(builtin, args, 2, 3, lineNumber); + return splitIntoArray( + args[0], + args[1], + args.length == 3 ? args[2] : jrt.getFSVar(), + lineNumber); + case SPRINTF: + requireIndirectArgumentCount(builtin, args, 1, Integer.MAX_VALUE, lineNumber); + return jrt + .getAwkSink() + .sprintf( + jrt.toAwkString(args[0]), + Arrays.copyOfRange(args, 1, args.length)); + case SQRT: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return Math.sqrt(JRT.toDouble(args[0])); + case SRAND: + requireIndirectArgumentCount(builtin, args, 0, 1, lineNumber); + int seed = args.length == 0 ? JRT.timeSeed() : (int) JRT.toDouble(args[0]); + int previousSeed = randomNumberGenerator.getSeed(); + randomNumberGenerator.setSeed(seed); + return Integer.valueOf(previousSeed); + case SUBSTR: + requireIndirectArgumentCount(builtin, args, 2, 3, lineNumber); + return substring( + args[0], + args[1], + args.length == 3 ? args[2] : null); + case SYSTEM: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return jrt.jrtSystem(jrt.toAwkString(args[0])); + case TOLOWER: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return jrt.toAwkString(args[0]).toLowerCase(); + case TOUPPER: + requireIndirectArgumentCount(builtin, args, 1, 1, lineNumber); + return jrt.toAwkString(args[0]).toUpperCase(); + default: + throw new AwkRuntimeException( + lineNumber, + "indirect calls are not implemented for builtin `" + builtin.getAwkName() + "'"); + } + } + + private void requireIndirectArgumentCount( + BuiltinFunction builtin, + Object[] args, + int minimum, + int maximum, + int lineNumber) { + if (args.length < minimum || args.length > maximum) { + String expected = minimum == maximum ? Integer.toString(minimum) : minimum + " to " + maximum; + throw new AwkRuntimeException( + lineNumber, + builtin.getAwkName() + " requires " + expected + " argument(s), not " + args.length); + } + } + + private Object invokeExtension( + ExtensionFunction function, + Object[] args, + int lineNumber, + boolean blockResult) { + // Let extensions report diagnostics at the call location. + currentLineNumber = lineNumber; + String extensionClassName = function.getExtensionClassName(); + JawkExtension extension = extensionInstances.get(extensionClassName); + if (extension == null) { + throw new AwkRuntimeException( + lineNumber, + "Extension instance for class '" + extensionClassName + "' is not registered"); + } + if (!(extension instanceof AbstractExtension)) { + throw new AwkRuntimeException( + lineNumber, + "Extension instance for class '" + extensionClassName + + "' does not extend " + + AbstractExtension.class.getName()); + } + Object result = function.invoke((AbstractExtension) extension, args); + if (blockResult && result instanceof BlockObject) { + result = new BlockManager().block((BlockObject) result); + } + if (result == null) { + return ""; + } + if (result instanceof Number + || result instanceof String + || result instanceof Map + || result instanceof BlockObject) { + return result; } + return jrt.toAwkString(result); } private void execMatch() { @@ -2811,13 +3046,19 @@ private void execMatch() { } private void execSubForDollar0(BooleanTuple tuple) { - boolean isGsub = tuple.getValue(); - String repl = jrt.toAwkString(pop()); - String ere = jrt.toAwkString(pop()); + Object replacement = pop(); + Object ere = pop(); + push(substituteInputLine(tuple.getValue(), ere, replacement)); + } + + private Object substituteInputLine(boolean global, Object ere, Object replacement) { String orig = jrt.toAwkString(jrt.jrtGetInputField(0)); - push(isGsub ? jrt.replaceAll(orig, repl, ere) : jrt.replaceFirst(orig, repl, ere)); + Object replacements = global ? + jrt.replaceAll(orig, jrt.toAwkString(replacement), jrt.toAwkString(ere)) : + jrt.replaceFirst(orig, jrt.toAwkString(replacement), jrt.toAwkString(ere)); jrt.setInputLine(jrt.getReplaceResult()); jrt.jrtParseFields(); + return replacements; } private void execSubForDollarReference(BooleanTuple tuple) { @@ -2868,49 +3109,53 @@ private void execSplit(CountTuple tuple, PositionTracker position) { } else { throw new Error("Invalid # of args. split() requires 2 or 3. Got: " + numArgs); } - Object o = pop(); - if (!(o instanceof Map)) { - throw new AwkRuntimeException(position.lineNumber(), o + " is not an array."); - } - String s = jrt.toAwkString(pop()); - Enumeration tokenizer = jrt.splitTokenizer(s, fs); + Object target = pop(); + Object source = pop(); + push(splitIntoArray(source, target, fs, position.lineNumber())); + } + private Object splitIntoArray(Object source, Object target, Object separator, int lineNumber) { + if (!(target instanceof Map)) { + throw new AwkRuntimeException(lineNumber, target + " is not an array."); + } + Enumeration tokenizer = jrt.splitTokenizer(jrt.toAwkString(source), separator); @SuppressWarnings("unchecked") - Map assocArray = (Map) o; + Map assocArray = (Map) target; assocArray.clear(); long cnt = 0; while (tokenizer.hasMoreElements()) { Object value = tokenizer.nextElement(); assocArray.put(++cnt, jrt.toInputScalar(value)); } - push(cnt); + return Long.valueOf(cnt); } private void execSubstr(CountTuple tuple) { long numArgs = tuple.getCount(); - int startPos, length; - String s; + Object length = null; if (numArgs == 3) { - length = (int) JRT.toLong(pop()); - startPos = (int) JRT.toDouble(pop()); - s = jrt.toAwkString(pop()); - } else if (numArgs == 2) { - startPos = (int) JRT.toDouble(pop()); - s = jrt.toAwkString(pop()); - length = s.length() - startPos + 1; - } else { + length = pop(); + } else if (numArgs != 2) { throw new Error("numArgs for SUBSTR must be 2 or 3. It is " + numArgs); } + Object start = pop(); + Object value = pop(); + push(substring(value, start, length)); + } + + private Object substring(Object value, Object start, Object requestedLength) { + String s = jrt.toAwkString(value); + int startPos = (int) JRT.toDouble(start); + int length = requestedLength == null ? + s.length() - startPos + 1 : (int) JRT.toLong(requestedLength); if (startPos <= 0) { startPos = 1; } if (length <= 0 || startPos > s.length()) { - push(BLANK); - } else if (startPos + length > s.length()) { - push(s.substring(startPos - 1)); - } else { - push(s.substring(startPos - 1, startPos + length - 1)); + return BLANK; } + return startPos + length > s.length() ? + s.substring(startPos - 1) : s.substring(startPos - 1, startPos + length - 1); } private void execSetNumGlobals(CountTuple tuple) { @@ -2999,9 +3244,8 @@ private void runBeforeStartHooks() { * SYMTAB is a gawk extension mirroring the symbol table: the script's * globals, the JRT-managed specials, and host-supplied variables that have * no compiled slot. The parser emits UPDATE_SYMTAB only when the script - * references SYMTAB outside POSIX mode. Values are a snapshot taken before - * execution; command-line name=value operand assignments update the array - * live, as in gawk. + * references SYMTAB outside POSIX mode. Declared globals and managed special + * variables remain linked to their runtime values. */ private void execUpdateSymtab(long offset) { symtabOffset = offset; @@ -3009,7 +3253,7 @@ private void execUpdateSymtab(long offset) { // a host-supplied SYMTAB value wins return; } - Map symtab = newAwkArray(); + SymtabArray symtab = new SymtabArray(); for (String name : executionInitialVariables.keySet()) { symtab.put(name, getVariable(name)); } @@ -3025,9 +3269,212 @@ private void execUpdateSymtab(long offset) { for (String name : getSpecialVariableNames()) { symtab.put(name, getVariable(name)); } + symtab.activate(); runtimeStack.setVariable(offset, symtab, true); } + private final class SymtabArray extends java.util.AbstractMap implements AssocArray { + private final Map entries = newAwkArray(); + private final Set assignableNames = new HashSet(); + private boolean active; + + private void activate() { + active = true; + } + + /** {@inheritDoc} */ + @Override + public Object get(Object key) { + String name = key == null ? "" : key.toString(); + if (active && !isMetaTableName(name) && isLiveSpecialVariable(name)) { + return getVariable(name); + } + Integer offset = globalVariableOffsets == null ? null : globalVariableOffsets.get(name); + if (active && !isMetaTableName(name) && offset != null) { + return runtimeStack.getVariable(offset.intValue(), true); + } + return entries.get(key); + } + + /** {@inheritDoc} */ + @Override + public boolean containsKey(Object key) { + String name = key == null ? "" : key.toString(); + return active + && !isMetaTableName(name) + && (isLiveSpecialVariable(name) + || (globalVariableOffsets != null && globalVariableOffsets.containsKey(name))) + || entries.containsKey(key); + } + + /** {@inheritDoc} */ + @Override + public Object put(Object key, Object value) { + String name = key == null ? "" : key.toString(); + if (!active) { + if (key != null) { + assignableNames.add(name); + } + return entries.put(key, value); + } + if (!assignableNames.contains(name)) { + throw new AwkRuntimeException("Cannot assign to an arbitrary element of SYMTAB."); + } + validateGlobalType(name, value); + Object previous = entries.put(key, value); + if (isMetaTableName(name)) { + return previous; + } + if (applyLiveSpecialVariable(name, value)) { + return previous; + } + Integer offset = globalVariableOffsets == null ? null : globalVariableOffsets.get(name); + if (offset != null) { + runtimeStack.setVariable(offset.intValue(), value, true); + } + return previous; + } + + private Object putRuntimeVariable(String name, Object value) { + assignableNames.add(name); + return put(name, value); + } + + private boolean applyLiveSpecialVariable(String name, Object value) { + if ("ARGC".equals(name)) { + if (argcOffset == NULL_OFFSET) { + throw new AwkRuntimeException("ARGC is read-only (not materialized)."); + } + runtimeStack.setVariable(argcOffset, value, true); + return true; + } + if ("RSTART".equals(name)) { + jrt.setRSTART(value); + return true; + } + if ("RLENGTH".equals(name)) { + jrt.setRLENGTH(value); + return true; + } + return isManagedSpecialVariable(name) && jrt.applySpecialVariable(name, value); + } + + private boolean isLiveSpecialVariable(String name) { + return isManagedSpecialVariable(name) + || "RSTART".equals(name) + || "RLENGTH".equals(name); + } + + /** {@inheritDoc} */ + @Override + public Object remove(Object key) { + if (active) { + throw new AwkRuntimeException("Cannot delete an element from SYMTAB."); + } + return entries.remove(key); + } + + /** {@inheritDoc} */ + @Override + public void clear() { + if (active) { + throw new AwkRuntimeException("Cannot delete SYMTAB."); + } + entries.clear(); + } + + private void validateGlobalType(String name, Object value) { + Boolean array = globalVariableArrays == null ? null : globalVariableArrays.get(name); + if (Boolean.TRUE.equals(array) && !(value instanceof Map)) { + throw new AwkRuntimeException( + "Attempting to use array `" + name + "' in a scalar context."); + } + if (Boolean.FALSE.equals(array) && value instanceof Map) { + throw new AwkRuntimeException( + "Attempting to use scalar `" + name + "' as an array."); + } + } + + /** {@inheritDoc} */ + @Override + public Set> entrySet() { + return new AbstractSet>() { + + @Override + public Iterator> iterator() { + final Iterator> iterator = entries.entrySet().iterator(); + return new Iterator>() { + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Map.Entry next() { + return new LiveSymtabEntry(iterator.next().getKey()); + } + + @Override + public void remove() { + if (active) { + throw new AwkRuntimeException("Cannot delete an element from SYMTAB."); + } + iterator.remove(); + } + }; + } + + @Override + public int size() { + return entries.size(); + } + }; + } + + private boolean isMetaTableName(String name) { + return "SYMTAB".equals(name) || "FUNCTAB".equals(name); + } + + private final class LiveSymtabEntry implements Map.Entry { + + private final Object key; + + private LiveSymtabEntry(Object key) { + this.key = key; + } + + @Override + public Object getKey() { + return key; + } + + @Override + public Object getValue() { + return SymtabArray.this.get(key); + } + + @Override + public Object setValue(Object value) { + return SymtabArray.this.put(key, value); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Map.Entry)) { + return false; + } + Map.Entry other = (Map.Entry) obj; + return Objects.equals(key, other.getKey()) && Objects.equals(getValue(), other.getValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(key) ^ Objects.hashCode(getValue()); + } + } + } + /* * FUNCTAB is a gawk extension listing the names of the standard built-in * functions, the program's user-defined functions, and the loaded @@ -3053,7 +3500,57 @@ private void execUpdateFunctab(long offset) { } } } - runtimeStack.setVariable(offset, functab, true); + runtimeStack.setVariable(offset, new ReadOnlyArray("FUNCTAB", functab), true); + } + + private static final class ReadOnlyArray extends java.util.AbstractMap implements AssocArray { + private final String name; + private final Map entries; + + private ReadOnlyArray(String nameParam, Map entriesParam) { + name = nameParam; + entries = entriesParam; + } + + /** {@inheritDoc} */ + @Override + public Object get(Object key) { + return JRT.containsAwkKey(entries, key) ? entries.get(key) : BLANK; + } + + /** {@inheritDoc} */ + @Override + public boolean containsKey(Object key) { + return JRT.containsAwkKey(entries, key); + } + + /** {@inheritDoc} */ + @Override + public Set> entrySet() { + return Collections.unmodifiableMap(entries).entrySet(); + } + + /** {@inheritDoc} */ + @Override + public Object put(Object key, Object value) { + throw readOnlyError(); + } + + /** {@inheritDoc} */ + @Override + public Object remove(Object key) { + throw readOnlyError(); + } + + /** {@inheritDoc} */ + @Override + public void clear() { + throw readOnlyError(); + } + + private AwkRuntimeException readOnlyError() { + return new AwkRuntimeException(name + " is read-only."); + } } /** Reflects a command-line variable assignment into a materialized SYMTAB. */ @@ -3062,7 +3559,9 @@ private void updateSymtabEntry(String name, Object value) { return; } Object symtab = runtimeStack.getVariable(symtabOffset, true); - if (symtab instanceof Map) { + if (symtab instanceof SymtabArray) { + ((SymtabArray) symtab).putRuntimeVariable(name, value); + } else if (symtab instanceof Map) { @SuppressWarnings("unchecked") Map symtabMap = (Map) symtab; symtabMap.put(name, value); @@ -3758,12 +4257,48 @@ private Object normalizeExternalVariableValue(Object value) { private static final UninitializedObject BLANK = new UninitializedObject(); + private static final class IndirectArgumentReference { + private final long offset; + private final boolean global; + private final Object scalarValue; + + private IndirectArgumentReference(long offsetParam, boolean globalParam, Object scalarValueParam) { + offset = offsetParam; + global = globalParam; + scalarValue = scalarValueParam; + } + } + + private static final class IndirectArrayArgumentReference { + private final Map map; + private final Object key; + private final Object scalarValue; + + private IndirectArrayArgumentReference( + Map mapParam, + Object keyParam, + Object scalarValueParam) { + map = mapParam; + key = keyParam; + scalarValue = scalarValueParam; + } + } + /** * Global names that must not participate in persistent memory even though they * are technically user-visible variables. */ private static final Set NON_PERSISTENT_GLOBALS = new HashSet<>( - Arrays.asList("ARGV", "ARGC", "ENVIRON", "RSTART", "RLENGTH", "IGNORECASE")); + Arrays + .asList( + "ARGV", + "ARGC", + "ENVIRON", + "RSTART", + "RLENGTH", + "IGNORECASE", + "SYMTAB", + "FUNCTAB")); private static final class SingleRecordInputSource implements InputSource { diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 2ce674a8..10bb90ba 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -26,8 +26,15 @@ import java.io.IOException; import java.io.LineNumberReader; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +42,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import io.jawk.AwkSandboxException; import io.jawk.NotImplementedError; import io.jawk.backend.AVM; import io.jawk.ext.ExtensionFunction; @@ -42,6 +50,8 @@ import io.jawk.intermediate.AwkTuples; import io.jawk.intermediate.BuiltinFunction; import io.jawk.jrt.JRT; +import io.jawk.intermediate.Tuple; +import io.jawk.util.ScriptFileSource; import io.jawk.util.ScriptSource; import io.jawk.frontend.ast.LexerException; import io.jawk.frontend.ast.ParserException; @@ -129,6 +139,10 @@ enum Token { EXTENSION, TYPED_REGEXP, + INDIRECT, + DIRECTIVE_INCLUDE, + DIRECTIVE_NAMESPACE, + DIRECTIVE_UNSUPPORTED, KW_FUNCTION, KW_BEGIN, @@ -244,6 +258,9 @@ enum Token { /** POSIX compile-time mode: rejects gawk syntax such as arrays of arrays and typed regexps. */ private final boolean posix; + /** Whether the compiling engine permits source inclusion from the filesystem. */ + private final boolean sourceIncludeAllowed; + /** *

* Constructor for AwkParser. @@ -253,8 +270,24 @@ enum Token { * @param posix {@code true} to enforce POSIX compile-time behavior */ public AwkParser(Map extensions, boolean posix) { - this.extensions = extensions == null ? Collections.emptyMap() : new HashMap<>(extensions); + this(extensions, posix, true); + } + + /** + * Creates a parser with an explicit source-inclusion policy. + * + * @param extensions extension functions available during parsing + * @param posix {@code true} to enforce POSIX compile-time behavior + * @param sourceIncludeAllowed {@code true} to permit {@code @include} + */ + public AwkParser( + Map extensions, + boolean posix, + boolean sourceIncludeAllowed) { + this.extensions = extensions == null ? + Collections.emptyMap() : Collections.unmodifiableMap(new HashMap<>(extensions)); this.posix = posix; + this.sourceIncludeAllowed = sourceIncludeAllowed; } /** @@ -270,16 +303,77 @@ private boolean isDisabledKeyword(Token keywordToken) { return posix && (keywordToken == Token.KW_BEGINFILE || keywordToken == Token.KW_ENDFILE); } + private boolean isAwkNamespaceIdentifier(String identifier) { + int separator = identifier.indexOf("::"); + if (separator >= 0) { + return "awk".equals(identifier.substring(0, separator)); + } + return "awk".equals(currentNamespace); + } + + private String awkNamespaceComponent(String identifier) { + return identifier.startsWith("awk::") ? identifier.substring("awk::".length()) : identifier; + } + + private String qualifyGlobalIdentifier(String identifier) { + int separator = identifier.indexOf("::"); + if (separator >= 0) { + return "awk".equals(identifier.substring(0, separator)) ? identifier.substring(separator + 2) : identifier; + } + if ("awk".equals(currentNamespace) || isAllUppercaseIdentifier(identifier)) { + return identifier; + } + return currentNamespace + "::" + identifier; + } + + private boolean isAllUppercaseIdentifier(String identifier) { + if (identifier.isEmpty()) { + return false; + } + for (int i = 0; i < identifier.length(); i++) { + char ch = identifier.charAt(i); + if (ch < 'A' || ch > 'Z') { + return false; + } + } + return true; + } + private List scriptSources; private int scriptSourcesCurrentIndex; + private ScriptSource currentScriptSource; private LineNumberReader reader; private int c; private Token token; + private String pendingIndirectIdentifier; + private boolean pendingColon; + private String currentNamespace = "awk"; + private final Deque includedSourceStack = new ArrayDeque(); + private final Set includedSourcePaths = new HashSet(); + private final Set topLevelSourcePaths = new HashSet(); private StringBuffer text = new StringBuffer(); private StringBuffer string = new StringBuffer(); private StringBuffer regexp = new StringBuffer(); + private static final class SourceState { + private final ScriptSource scriptSource; + private final LineNumberReader reader; + private final int currentCharacter; + private final String namespace; + + private SourceState( + ScriptSource scriptSourceParam, + LineNumberReader readerParam, + int currentCharacterParam, + String namespaceParam) { + scriptSource = scriptSourceParam; + reader = readerParam; + currentCharacter = currentCharacterParam; + namespace = namespaceParam; + } + } + private void read() throws IOException { text.append((char) c); c = reader.read(); @@ -287,10 +381,40 @@ private void read() throws IOException { while (c == '\r') { c = reader.read(); } - if (c < 0 && (scriptSourcesCurrentIndex + 1) < scriptSources.size()) { - scriptSourcesCurrentIndex++; - reader = new LineNumberReader(scriptSources.get(scriptSourcesCurrentIndex).getReader()); - read(); + } + + /** + * Advances to the next readable source at a token boundary when the current + * reader has reached end-of-file. Deferring this transition until the current + * token is complete preserves the source and namespace used to classify its + * final token. Included files are unwound first in LIFO order, restoring the + * including reader, its unread character, and its namespace. Once the include + * stack is empty, parsing continues with the next top-level script source, + * whose namespace starts at {@code awk}. + * + * @throws IOException if a source cannot be closed or read + */ + private void advancePastEndOfSource() throws IOException { + while (c < 0) { + if (!includedSourceStack.isEmpty()) { + reader.close(); + SourceState previous = includedSourceStack.pop(); + currentScriptSource = previous.scriptSource; + reader = previous.reader; + c = previous.currentCharacter; + currentNamespace = previous.namespace; + } else if ((scriptSourcesCurrentIndex + 1) < scriptSources.size()) { + scriptSourcesCurrentIndex++; + currentScriptSource = scriptSources.get(scriptSourcesCurrentIndex); + reader = new LineNumberReader(currentScriptSource.getReader()); + currentNamespace = "awk"; + c = reader.read(); + while (c == '\r') { + c = reader.read(); + } + } else { + return; + } } } @@ -324,7 +448,13 @@ public AstNode parse(List localScriptSources) throws IOException { } this.scriptSources = Collections.unmodifiableList(new ArrayList<>(localScriptSources)); scriptSourcesCurrentIndex = 0; - reader = new LineNumberReader(this.scriptSources.get(scriptSourcesCurrentIndex).getReader()); + currentScriptSource = this.scriptSources.get(scriptSourcesCurrentIndex); + reader = new LineNumberReader(currentScriptSource.getReader()); + currentNamespace = "awk"; + includedSourceStack.clear(); + resetIncludedSourcePaths(); + pendingIndirectIdentifier = null; + pendingColon = false; read(); lexer(); return SCRIPT(); @@ -347,7 +477,13 @@ public AstNode parseExpression(ScriptSource expressionSource) throws IOException // Reader of the expression this.scriptSources = Collections.singletonList(expressionSource); scriptSourcesCurrentIndex = 0; - reader = new LineNumberReader(this.scriptSources.get(scriptSourcesCurrentIndex).getReader()); + currentScriptSource = expressionSource; + reader = new LineNumberReader(currentScriptSource.getReader()); + currentNamespace = "awk"; + includedSourceStack.clear(); + resetIncludedSourcePaths(); + pendingIndirectIdentifier = null; + pendingColon = false; // Initialize the lexer read(); @@ -357,10 +493,23 @@ public AstNode parseExpression(ScriptSource expressionSource) throws IOException return EXPRESSION_TO_EVALUATE(); } + private void resetIncludedSourcePaths() throws IOException { + includedSourcePaths.clear(); + topLevelSourcePaths.clear(); + for (ScriptSource source : scriptSources) { + if (source instanceof ScriptFileSource) { + String filePath = ((ScriptFileSource) source).getFilePath(); + Path sourcePath = Paths.get(filePath).toRealPath(); + includedSourcePaths.add(sourcePath); + topLevelSourcePaths.add(sourcePath); + } + } + } + private LexerException lexerException(String msg) { return new LexerException( msg, - scriptSources.get(scriptSourcesCurrentIndex).getDescription(), + currentScriptSource.getDescription(), reader.getLineNumber()); } @@ -553,7 +702,11 @@ private Token lexer(Token expectedToken) throws IOException { private Token lexer() throws IOException { // clear whitespace - while (c >= 0 && (c == ' ' || c == '\t' || c == '#' || c == '\\')) { + while (true) { + advancePastEndOfSource(); + if (c < 0 || c != ' ' && c != '\t' && c != '#' && c != '\\') { + break; + } if (c == '\\') { read(); if (c == '\n') { @@ -571,6 +724,17 @@ private Token lexer() throws IOException { } } text.setLength(0); + if (pendingColon) { + pendingColon = false; + token = Token.COLON; + return token; + } + if (pendingIndirectIdentifier != null) { + text.append(pendingIndirectIdentifier); + pendingIndirectIdentifier = null; + token = Token.ID; + return token; + } if (c < 0) { token = Token.EOF; return token; @@ -619,15 +783,52 @@ private Token lexer() throws IOException { } if (c == '@') { if (posix) { - throw lexerException("Typed regular expressions are not supported in POSIX mode."); + throw lexerException("gawk @ syntax is not supported in POSIX mode."); } read(); - if (c != '/') { - throw lexerException("Invalid character (64): @"); + if (c == '/') { + read(); + readRegexp(); + token = Token.TYPED_REGEXP; + return token; } - read(); - readRegexp(); - token = Token.TYPED_REGEXP; + if (Character.isJavaIdentifierStart(c)) { + while (Character.isJavaIdentifierPart(c)) { + read(); + } + if (c == ':') { + read(); + if (c != ':') { + throw lexerException("Namespace separator must be two colons (::)."); + } + read(); + if (!Character.isJavaIdentifierStart(c)) { + throw lexerException("A namespace-qualified name requires an identifier after ::."); + } + read(); + while (Character.isJavaIdentifierPart(c)) { + read(); + } + } + String atWord = text.toString(); + if ("@include".equals(atWord)) { + token = Token.DIRECTIVE_INCLUDE; + return token; + } + if ("@namespace".equals(atWord)) { + token = Token.DIRECTIVE_NAMESPACE; + return token; + } + if ("@load".equals(atWord)) { + token = Token.DIRECTIVE_UNSUPPORTED; + return token; + } + pendingIndirectIdentifier = atWord.substring(1); + validateIndirectIdentifier(pendingIndirectIdentifier); + token = Token.INDIRECT; + return token; + } + token = Token.INDIRECT; return token; } if (c == '~') { @@ -838,22 +1039,68 @@ private Token lexer() throws IOException { while (Character.isJavaIdentifierPart(c)) { read(); } + if (c == ':') { + read(); + if (c != ':') { + text.setLength(text.length() - 1); + pendingColon = true; + } else { + if (posix) { + throw lexerException("gawk namespace syntax is not supported in POSIX mode."); + } + read(); + if (!Character.isJavaIdentifierStart(c)) { + throw lexerException("A namespace-qualified name requires an identifier after ::."); + } + read(); + while (Character.isJavaIdentifierPart(c)) { + read(); + } + if (c == ':') { + read(); + if (c == ':') { + throw lexerException("A namespace-qualified name may contain only one :: separator."); + } + text.setLength(text.length() - 1); + pendingColon = true; + } + } + } // check for certain keywords // extensions override built-in stuff - if (extensions.get(text.toString()) != null) { + String sourceIdentifier = text.toString(); + int namespaceSeparator = sourceIdentifier.indexOf("::"); + if (namespaceSeparator >= 0) { + String namespaceComponent = sourceIdentifier.substring(namespaceSeparator + 2); + if (KEYWORDS.containsKey(namespaceComponent) + || BuiltinFunction.of(namespaceComponent) != null) { + throw lexerException( + "Reserved word cannot be used after a namespace separator: " + + sourceIdentifier); + } + } + String lookupIdentifier = awkNamespaceComponent(sourceIdentifier); + boolean awkNamespaceIdentifier = isAwkNamespaceIdentifier(sourceIdentifier); + boolean unqualifiedIdentifier = namespaceSeparator < 0; + if (awkNamespaceIdentifier && extensions.get(lookupIdentifier) != null) { + text.setLength(0); + text.append(lookupIdentifier); token = Token.EXTENSION; return token; } - Token kwToken = KEYWORDS.get(text.toString()); + Token kwToken = KEYWORDS.get(sourceIdentifier); if (kwToken != null && !isDisabledKeyword(kwToken)) { token = kwToken; return token; } - if (BuiltinFunction.of(text.toString()) != null) { + if ((unqualifiedIdentifier || awkNamespaceIdentifier) + && BuiltinFunction.of(lookupIdentifier) != null) { + text.setLength(0); + text.append(lookupIdentifier); token = Token.BUILTIN_FUNC_NAME; return token; } - if (c == '(') { + if (c == '(' && !pendingColon) { token = Token.FUNC_ID; return token; } else { @@ -977,7 +1224,15 @@ AST EXPRESSION_TO_EVALUATE() throws IOException { AST RULE_LIST() throws IOException { optNewline(); AST ruleOrFunction = null; - if (token == Token.KW_FUNCTION) { + if (token == Token.DIRECTIVE_INCLUDE) { + INCLUDE_DIRECTIVE(); + return RULE_LIST(); + } else if (token == Token.DIRECTIVE_NAMESPACE) { + NAMESPACE_DIRECTIVE(); + return RULE_LIST(); + } else if (token == Token.DIRECTIVE_UNSUPPORTED) { + throw parserException("Unsupported gawk directive: " + text); + } else if (token == Token.KW_FUNCTION) { ruleOrFunction = FUNCTION(); } else if (token != Token.EOF) { ruleOrFunction = RULE(); @@ -988,12 +1243,144 @@ AST RULE_LIST() throws IOException { return new RuleListAst(ruleOrFunction, RULE_LIST()); } + private void NAMESPACE_DIRECTIVE() throws IOException { + lexer(); + if (token != Token.STRING) { + throw parserException("@namespace requires a quoted namespace name."); + } + String namespace = string.toString(); + validateNamespace(namespace); + currentNamespace = namespace; + lexer(); + terminator(); + } + + private void INCLUDE_DIRECTIVE() throws IOException { + lexer(); + if (token != Token.STRING) { + throw parserException("@include requires a quoted file name."); + } + if (!sourceIncludeAllowed) { + throw new AwkSandboxException("@include is disabled in sandbox mode"); + } + String includeName = string.toString(); + boolean includeTerminatedByEndOfSource = validateIncludeTerminator(); + Path includePath = resolveIncludePath(includeName); + if (topLevelSourcePaths.contains(includePath)) { + throw parserException( + "Cannot include a top-level program source: " + includeName); + } + if (!includedSourcePaths.add(includePath)) { + lexer(); + if (!includeTerminatedByEndOfSource) { + terminator(); + } + return; + } + includedSourceStack.push(new SourceState(currentScriptSource, reader, c, currentNamespace)); + currentScriptSource = new ScriptSource( + includePath.toString(), + Files.newBufferedReader(includePath, StandardCharsets.UTF_8)); + reader = new LineNumberReader(currentScriptSource.getReader()); + currentNamespace = "awk"; + c = reader.read(); + while (c == '\r') { + c = reader.read(); + } + advancePastEndOfSource(); + lexer(); + } + + private boolean validateIncludeTerminator() throws IOException { + while (c == ' ' || c == '\t') { + read(); + } + if (c == '#') { + while (c >= 0 && c != '\n') { + read(); + } + } + if (c >= 0 && c != '\n' && c != ';') { + throw parserException("@include must be followed by a newline, semicolon, or end of file."); + } + return c < 0; + } + + private void validateNamespace(String namespace) { + if (namespace == null + || namespace.isEmpty() + || !Character.isJavaIdentifierStart(namespace.charAt(0))) { + throw parserException("Invalid gawk namespace name: " + namespace); + } + for (int i = 1; i < namespace.length(); i++) { + if (!Character.isJavaIdentifierPart(namespace.charAt(i))) { + throw parserException("Invalid gawk namespace name: " + namespace); + } + } + if (KEYWORDS.containsKey(namespace) + || BuiltinFunction.of(namespace) != null + || extensions.containsKey(namespace)) { + throw parserException("Reserved identifier cannot be used as a gawk namespace: " + namespace); + } + } + + private void validateIndirectIdentifier(String identifier) throws LexerException { + int separator = identifier.indexOf("::"); + String namespace = separator < 0 ? "awk" : identifier.substring(0, separator); + String component = separator < 0 ? identifier : identifier.substring(separator + 2); + if (KEYWORDS.containsKey(component) + || BuiltinFunction.of(component) != null + || ("awk".equals(namespace) && extensions.containsKey(component))) { + throw lexerException("Reserved identifier cannot be used as an indirect-call selector: " + identifier); + } + } + + private Path resolveIncludePath(String includeName) { + Path requested = Paths.get(includeName); + List candidates = new ArrayList(); + if (requested.isAbsolute()) { + candidates.add(requested); + } else { + if (!ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT.equals(currentScriptSource.getDescription())) { + try { + Path sourcePath = Paths.get(currentScriptSource.getDescription()); + Path parent = sourcePath.toAbsolutePath().normalize().getParent(); + if (parent != null) { + candidates.add(parent.resolve(requested)); + } + } catch (InvalidPathException ignored) { + // Reader-backed ScriptSource values may use a descriptive + // label rather than a file path. + } + } + String awkPath = System.getenv("AWKPATH"); + if (awkPath != null) { + for (String entry : awkPath.split(java.util.regex.Pattern.quote(File.pathSeparator), -1)) { + candidates.add(Paths.get(entry.isEmpty() ? "." : entry).resolve(requested)); + } + } + candidates.add(requested); + } + for (Path candidate : candidates) { + Path normalized = candidate.toAbsolutePath().normalize(); + if (Files.isRegularFile(normalized)) { + try { + return normalized.toRealPath(); + } catch (IOException ignored) { + // The candidate may have disappeared between the existence + // check and canonicalization; continue searching AWKPATH. + } + } + } + throw parserException("Cannot find @include file: " + includeName); + } + // FUNCTION: function functionName( [FORMAL_PARAM_LIST] ) STATEMENT_LIST AST FUNCTION() throws IOException { expectKeyword("function"); String functionName; if (token == Token.FUNC_ID || token == Token.ID) { - functionName = text.toString(); + functionName = qualifyGlobalIdentifier(text.toString()); lexer(); } else { throw parserException("Expecting function name. Got " + token.name() + ": " + text); @@ -1497,6 +1884,8 @@ AST FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allowMultidi AST str = symbolTable.addSTRING(string.toString()); lexer(); return str; + } else if (token == Token.INDIRECT) { + return INDIRECT_FUNCTION_CALL(allowInKeyword); } else if (token == Token.TYPED_REGEXP) { AST regexpAst = symbolTable.addTYPED_REGEXP(regexp.toString()); lexer(); @@ -1527,6 +1916,19 @@ AST FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allowMultidi } } + private AST INDIRECT_FUNCTION_CALL(boolean allowInKeyword) throws IOException { + lexer(); + if (token != Token.ID) { + throw parserException("An indirect function call requires a variable name after @."); + } + AST functionNameAst = symbolTable.getID(text.toString()); + lexer(); + lexer(Token.OPEN_PAREN); + AST params = token == Token.CLOSE_PAREN ? null : EXPRESSION_LIST(true, allowInKeyword); + lexer(Token.CLOSE_PAREN); + return new IndirectFunctionCallAst(functionNameAst, params); + } + // SYMBOL : Token.ID [ '(' params ')' | '[' ASSIGNMENT_EXPRESSION ']' ] AST SYMBOL(boolean allowComparison, boolean allowInKeyword) throws IOException { if (token != Token.ID && token != Token.FUNC_ID && token != Token.BUILTIN_FUNC_NAME && token != Token.EXTENSION) { @@ -2288,6 +2690,28 @@ private void populateRawRegexpParameterTuples(AST valueAst, AwkTuples tuples) { valueAst.populateTuples(tuples); } + private int populateIndirectActualParameters( + AwkTuples tuples, + FunctionCallParamListAst params) { + if (params == null) { + return 0; + } + AST argument = params.getAst1(); + if (argument instanceof IDAst + && !isJrtManagedSpecialName(((IDAst) argument).id)) { + IDAst idAst = (IDAst) argument; + tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal); + } else if (argument instanceof ArrayReferenceAst) { + ((ArrayReferenceAst) argument).populateTargetReferenceTuples(tuples); + tuples.pushIndirectArrayArgument(); + } else { + argument.populateTuples(tuples); + } + return 1 + populateIndirectActualParameters( + tuples, + (FunctionCallParamListAst) params.getAst2()); + } + private Set collectArrayParameterIndexes(FunctionDefAst functionDefAst) { Set arrayIndexes = new HashSet(); FunctionDefParamListAst fPtr = (FunctionDefParamListAst) functionDefAst.getAst1(); @@ -2308,7 +2732,7 @@ private Set collectArrayParameterIndexes(FunctionDefAst functionDefAst) // AST class defs private abstract class AST extends AstNode { - private final String sourceDescription = scriptSources.get(scriptSourcesCurrentIndex).getDescription(); + private final String sourceDescription = currentScriptSource.getDescription(); // PositionTracker consumes these tuple-emitted source lines at runtime, but // AST nodes have to capture them here during parsing before tuples exist. private final int lineNo; @@ -4255,6 +4679,31 @@ private int warningLineNo() { } + private final class IndirectFunctionCallAst extends ScalarExpressionAst { + + private IndirectFunctionCallAst(AST functionNameAst, AST params) { + super(functionNameAst, params); + } + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + getAst1().populateTuples(tuples); + int actualParamCount = populateIndirectActualParameters( + tuples, + (FunctionCallParamListAst) getAst2()); + tuples + .indirectCall( + symbolTable.indirectFunctionTargets(), + extensions, + actualParamCount, + sourceBasename(), + getLineNo()); + popSourceLineNumber(tuples); + return 1; + } + } + private final class BuiltinFunctionCallAst extends ScalarExpressionAst { private final String id; @@ -5873,6 +6322,7 @@ int numGlobals() { // functions (proxies) private Map functionProxies = new HashMap(); + private Map cachedIndirectFunctionTargets; // variable management private Map globalIds = new HashMap(); @@ -5930,12 +6380,7 @@ private boolean isGlobalReferenced(String id) { } private IDAst getID(String id) { - if (functionProxies.get(id) != null) { - throw parserException("cannot use " + id + " as a variable; it is a function"); - } - - // put in the pool of ids to guard against using it as a function name - ids.add(id); + id = resolveVariableIdentifier(id); Map map; if (currentFunctionName == null) { @@ -5955,6 +6400,15 @@ private IDAst getID(String id) { map = globalIds; } } + if (map == globalIds) { + // Only global variables share the namespace with function names. + // Formal parameters may legitimately have the same name as an + // unrelated function. + if (functionProxies.get(id) != null) { + throw parserException("cannot use " + id + " as a variable; it is a function"); + } + ids.add(id); + } IDAst idAst = map.get(id); if (idAst == null) { idAst = new IDAst(id, map == globalIds); @@ -5969,6 +6423,16 @@ private IDAst getID(String id) { return idAst; } + private String resolveVariableIdentifier(String id) { + if (currentFunctionName != null) { + Set parameters = functionParameters.get(currentFunctionName); + if (parameters != null && parameters.contains(id)) { + return id; + } + } + return qualifyGlobalIdentifier(id); + } + AST addID(String id) throws ParserException { IDAst retVal = getID(id); retVal.markReferenced(); @@ -5983,6 +6447,12 @@ AST addID(String id) throws ParserException { } int addFunctionParameter(String functionName, String id) { + int namespaceSeparator = functionName.indexOf("::"); + String unqualifiedFunctionName = namespaceSeparator < 0 ? + functionName : functionName.substring(namespaceSeparator + 2); + if (unqualifiedFunctionName.equals(id)) { + throw parserException("cannot use " + id + " as a parameter; it is the function name"); + } Set set = functionParameters.get(functionName); if (set == null) { set = new HashSet(); @@ -6037,6 +6507,7 @@ AST addFunctionDef(String functionName, AST paramList, AST block) { } AST addFunctionCall(String id, AST paramList) { + id = qualifyGlobalIdentifier(id); FunctionProxy functionProxy = functionProxies.get(id); if (functionProxy == null) { functionProxy = new FunctionProxy(id); @@ -6045,6 +6516,27 @@ AST addFunctionCall(String id, AST paramList) { return new FunctionCallAst(functionProxy, paramList); } + Map indirectFunctionTargets() { + if (cachedIndirectFunctionTargets != null) { + return cachedIndirectFunctionTargets; + } + Map targets = new HashMap(); + for (Map.Entry entry : functionProxies.entrySet()) { + FunctionProxy proxy = entry.getValue(); + if (proxy.isDefined()) { + targets + .put( + entry.getKey(), + new Tuple.IndirectFunctionTarget( + proxy, + proxy.getFunctionParamCount(), + collectArrayParameterIndexes(proxy.functionDefAst))); + } + } + cachedIndirectFunctionTargets = Collections.unmodifiableMap(targets); + return cachedIndirectFunctionTargets; + } + AST addArrayReference(String id, AST idxAst, int lineNo) throws ParserException { return new ArrayReferenceAst(lineNo, addArrayID(id), idxAst); } @@ -6076,7 +6568,7 @@ AST addTYPED_REGEXP(String localRegexp) { private ParserException parserException(String msg) { return new ParserException( msg, - scriptSources.get(scriptSourcesCurrentIndex).getDescription(), + currentScriptSource.getDescription(), reader.getLineNumber()); } } diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index f8b536c7..10d0fda4 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -433,6 +433,24 @@ public void peekDereference(int offset, boolean isGlobal) { queue.add(new Tuple.VariableTuple(Opcode.PEEK_DEREFERENCE, offset, isGlobal)); } + /** + * Emits a variable reference whose scalar value is captured immediately while + * retaining the variable location for a runtime-selected array parameter. + * + * @param offset variable offset + * @param isGlobal whether the variable is global + */ + public void pushIndirectArgument(int offset, boolean isGlobal) { + queue.add(new Tuple.VariableTuple(Opcode.PUSH_INDIRECT_ARGUMENT, offset, isGlobal)); + } + + /** + * Emits an indirect-call subarray argument after its containing map and key. + */ + public void pushIndirectArrayArgument() { + queue.add(new Tuple.NoOperandTuple(Opcode.PUSH_INDIRECT_ARRAY_ARGUMENT)); + } + /** *

* plusEq. @@ -1681,6 +1699,31 @@ public void callFunction( queue.add(new Tuple.CallFunctionTuple(addressSupplier, funcName, numFormalParams, numActualParams)); } + /** + * Emits a call whose function name is evaluated at runtime. + * + * @param userFunctions available user-defined function targets + * @param extensionFunctions available extension function targets + * @param numActualParams number of evaluated actual parameters + * @param sourceName source name for runtime diagnostics + * @param lineNumber source line for runtime diagnostics + */ + public void indirectCall( + Map userFunctions, + Map extensionFunctions, + int numActualParams, + String sourceName, + int lineNumber) { + queue + .add( + new Tuple.IndirectCallTuple( + userFunctions, + extensionFunctions, + numActualParams, + sourceName, + lineNumber)); + } + /** * Emits a tuple that prints a diagnostic message to the warning stream when * executed. Planted by the parser just before the instruction it describes, @@ -2233,8 +2276,7 @@ private boolean peepholeOptimizePass() { private boolean[] addressTargets(List tuples, int tupleCount) { boolean[] targets = new boolean[tupleCount]; for (Tuple tuple : tuples) { - Address address = tuple.getAddress(); - if (address != null) { + for (Address address : tuple.getAddresses()) { int index = address.index(); if (index >= 0 && index < tupleCount) { targets[index] = true; @@ -2454,7 +2496,9 @@ private void remapAddresses(int[] indexMapping) { } Set

processedAddresses = Collections.newSetFromMap(new IdentityHashMap()); for (Tuple tuple : queue) { - remapAddress(tuple.getAddress(), indexMapping, processedAddresses); + for (Address address : tuple.getAddresses()) { + remapAddress(address, indexMapping, processedAddresses); + } } // Property addresses may not be referenced by any tuple (e.g. after // jump threading rewired the loop-back GOTO), so they must be @@ -2696,8 +2740,7 @@ private void optimizeQueue() { } } - Address address = tuple.getAddress(); - if (address != null) { + for (Address address : tuple.getAddresses()) { int targetIndex = address.index(); if (targetIndex < 0 || targetIndex >= size) { throw new Error("address " + address + " doesn't resolve to an actual list element"); @@ -2732,8 +2775,7 @@ private void optimizeQueue() { } } - Address address = tuple.getAddress(); - if (address != null) { + for (Address address : tuple.getAddresses()) { int targetIndex = address.index(); if (targetIndex < 0 || targetIndex >= size) { throw new Error("address " + address + " doesn't resolve to an actual list element"); @@ -2926,6 +2968,7 @@ private boolean requiresEvalGlobalFrame(Opcode opcode) { case ASSIGN_ARRAY: case DEREFERENCE: case PEEK_DEREFERENCE: + case PUSH_INDIRECT_ARGUMENT: case PLUS_EQ: case MINUS_EQ: case MULT_EQ: @@ -2939,6 +2982,7 @@ private boolean requiresEvalGlobalFrame(Opcode opcode) { case MOD_EQ_ARRAY: case POW_EQ_ARRAY: case CALL_FUNCTION: + case INDIRECT_CALL: // extension calls read globals (e.g. IGNORECASE) and their // beforeStart hooks assign gawk-owned arrays case EXTENSION: diff --git a/src/main/java/io/jawk/intermediate/Opcode.java b/src/main/java/io/jawk/intermediate/Opcode.java index 80fadff2..e6d94c06 100644 --- a/src/main/java/io/jawk/intermediate/Opcode.java +++ b/src/main/java/io/jawk/intermediate/Opcode.java @@ -1605,7 +1605,26 @@ public enum Opcode { * Stack before: ...
* Stack after: argind-value ... */ - PUSH_ARGIND; + PUSH_ARGIND, + + /** + * Call a user-defined, built-in, or extension function selected by name at + * runtime. New opcodes are appended to preserve serialized numeric identifiers. + */ + INDIRECT_CALL, + + /** + * Push an indirect-call argument that snapshots its scalar value while retaining + * its variable location. The target selected at runtime determines whether to + * use the scalar snapshot or materialize/read an array at that location. + */ + PUSH_INDIRECT_ARGUMENT, + + /** + * Push an indirect-call subarray argument that snapshots its scalar value while + * retaining its containing map and key for a runtime-selected array parameter. + */ + PUSH_INDIRECT_ARRAY_ARGUMENT; private static final Opcode[] VALUES = values(); diff --git a/src/main/java/io/jawk/intermediate/Tuple.java b/src/main/java/io/jawk/intermediate/Tuple.java index 3f1412e9..b35f46a6 100644 --- a/src/main/java/io/jawk/intermediate/Tuple.java +++ b/src/main/java/io/jawk/intermediate/Tuple.java @@ -23,9 +23,15 @@ */ import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import java.util.regex.Pattern; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.jawk.ext.ExtensionFunction; /** @@ -66,20 +72,31 @@ public Address getAddress() { } /** - * Resolves deferred operands and validates resolved addresses. + * Returns every jump/call address carried by this tuple. * - * @param queue tuple queue used to validate address targets + * @return tuple addresses, or an empty list */ - public void touch(List queue) { + public List
getAddresses() { Address address = getAddress(); if (address == null) { - return; - } - if (address.index() == -1) { - throw new Error("address " + address + " is unresolved"); + return Collections.emptyList(); } - if (address.index() >= queue.size()) { - throw new Error("address " + address + " doesn't resolve to an actual list element"); + return Collections.singletonList(address); + } + + /** + * Resolves deferred operands and validates resolved addresses. + * + * @param queue tuple queue used to validate address targets + */ + public void touch(List queue) { + for (Address address : getAddresses()) { + if (address.index() == -1) { + throw new Error("address " + address + " is unresolved"); + } + if (address.index() >= queue.size()) { + throw new Error("address " + address + " doesn't resolve to an actual list element"); + } } } @@ -724,6 +741,165 @@ public String toString() { } } + /** + * Runtime target metadata for a user-defined indirect function call. + */ + public static final class IndirectFunctionTarget implements Serializable { + private static final long serialVersionUID = 1L; + private transient Supplier
addressSupplier; + private Address address; + private final long numFormalParams; + private final Set arrayParameterIndexes; + + /** + * Creates target metadata whose address is resolved during tuple + * post-processing. + * + * @param addressSupplierParam function entry-point supplier + * @param numFormalParamsParam formal parameter count + * @param arrayParameterIndexesParam zero-based array parameter indexes + */ + public IndirectFunctionTarget( + Supplier
addressSupplierParam, + long numFormalParamsParam, + Set arrayParameterIndexesParam) { + addressSupplier = addressSupplierParam; + numFormalParams = numFormalParamsParam; + arrayParameterIndexes = Collections.unmodifiableSet(new HashSet(arrayParameterIndexesParam)); + } + + private void resolve() { + if (address == null && addressSupplier != null) { + address = addressSupplier.get(); + addressSupplier = null; + } + } + + /** + * Returns the resolved function entry point. + * + * @return function address + */ + public Address getAddress() { + resolve(); + return address; + } + + /** + * Returns the function's formal parameter count. + * + * @return formal parameter count + */ + public long getNumFormalParams() { + return numFormalParams; + } + + /** + * Returns whether the parameter at the supplied index is an array. + * + * @param index zero-based formal parameter index + * @return {@code true} when the formal parameter is an array + */ + public boolean isArrayParameter(int index) { + return arrayParameterIndexes.contains(Integer.valueOf(index)); + } + } + + /** + * Tuple for a function call whose target name is evaluated at runtime. + */ + public static final class IndirectCallTuple extends Tuple { + private static final long serialVersionUID = 1L; + private final Map userFunctions; + private final Map extensionFunctions; + private final long numActualParams; + private final String sourceName; + private final int sourceLine; + + IndirectCallTuple( + Map userFunctionsParam, + Map extensionFunctionsParam, + long numActualParamsParam, + String sourceNameParam, + int sourceLineParam) { + super(Opcode.INDIRECT_CALL); + userFunctions = userFunctionsParam; + extensionFunctions = extensionFunctionsParam; + numActualParams = numActualParamsParam; + sourceName = sourceNameParam; + sourceLine = sourceLineParam; + } + + @Override + public void touch(List queue) { + for (IndirectFunctionTarget target : userFunctions.values()) { + target.resolve(); + } + super.touch(queue); + } + + @Override + public List
getAddresses() { + List
addresses = new ArrayList
(userFunctions.size()); + for (IndirectFunctionTarget target : userFunctions.values()) { + addresses.add(target.getAddress()); + } + return addresses; + } + + /** + * Returns the user-defined functions available to the call site. + * + * @return function target map + */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "The map is an immutable metadata snapshot shared by every indirect call tuple") + public Map getUserFunctions() { + return userFunctions; + } + + /** + * Returns the extension functions available to the call site. + * + * @return extension function map + */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "The map is an immutable metadata snapshot shared by every indirect call tuple") + public Map getExtensionFunctions() { + return extensionFunctions; + } + + /** + * Returns the number of actual parameters evaluated by the call site. + * + * @return actual parameter count + */ + public long getNumActualParams() { + return numActualParams; + } + + /** + * Returns the source name for runtime diagnostics. + * + * @return source name + */ + public String getSourceName() { + return sourceName; + } + + /** + * Returns the source line for runtime diagnostics. + * + * @return source line + */ + public int getSourceLine() { + return sourceLine; + } + + @Override + public String toString() { + return getOpcode().name() + ", " + numActualParams; + } + } + /** * Tuple for extension function invocations. */ diff --git a/src/main/java/io/jawk/jrt/BSDRandom.java b/src/main/java/io/jawk/jrt/BSDRandom.java index 8f0e0006..79435ad4 100644 --- a/src/main/java/io/jawk/jrt/BSDRandom.java +++ b/src/main/java/io/jawk/jrt/BSDRandom.java @@ -33,6 +33,7 @@ public class BSDRandom { private final int[] state = new int[RAND_DEG]; private int fptr; private int rptr; + private int seed; /** * Creates a new generator with the specified seed. @@ -47,13 +48,15 @@ public BSDRandom(int seed) { * Seed the generator. A seed of {@code 0} is transformed to {@code 1} * as in the original implementation. * - * @param seed New pseudo-random seed + * @param newSeed New pseudo-random seed */ - public final void setSeed(int seed) { - if (seed == 0) { - seed = 1; + public final void setSeed(int newSeed) { + seed = newSeed; + int effectiveSeed = newSeed; + if (effectiveSeed == 0) { + effectiveSeed = 1; } - state[0] = seed; + state[0] = effectiveSeed; for (int i = 1; i < RAND_DEG; i++) { long val = 16807L * state[i - 1] % 2147483647L; state[i] = (int) val; @@ -65,6 +68,15 @@ public final void setSeed(int seed) { } } + /** + * Returns the seed most recently supplied to {@link #setSeed(int)}. + * + * @return Current pseudo-random seed + */ + public int getSeed() { + return seed; + } + private int nextInt() { int val = state[fptr] + state[rptr]; state[fptr] = val; diff --git a/src/main/java/io/jawk/jrt/StreamInputSource.java b/src/main/java/io/jawk/jrt/StreamInputSource.java index f17e3048..49a9a12e 100644 --- a/src/main/java/io/jawk/jrt/StreamInputSource.java +++ b/src/main/java/io/jawk/jrt/StreamInputSource.java @@ -621,6 +621,9 @@ private void setFilelistVariable(String nameValue) { "Must have a non-blank variable name in a name=value variable assignment argument."); } String name = nameValue.substring(0, eqIdx); + if (name.startsWith("awk::")) { + name = name.substring("awk::".length()); + } String value = nameValue.substring(eqIdx + 1); vm.assignVariable(name, jrt.toInputScalar(value)); } diff --git a/src/site/markdown/cli-reference.md b/src/site/markdown/cli-reference.md index 0f1e06b0..b2bb5931 100644 --- a/src/site/markdown/cli-reference.md +++ b/src/site/markdown/cli-reference.md @@ -52,7 +52,7 @@ java -jar jawk-${project.version}-standalone.jar --list-ext > - `-r` disables Jawk's default trapping of `IllegalFormatException` for `printf` and `sprintf`. > - `--locale ` sets the locale through `Locale.forLanguageTag(...)`. > - `-t` keeps associative array keys sorted. -> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, typed regexp literals (`@/re/`), and the `BEGINFILE` / `ENDFILE` special patterns. +> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, all gawk `@` forms, and the `BEGINFILE` / `ENDFILE` special patterns. > - `JAWK_PERSISTENT_MEMORY` can also point at the persistent-memory file when you do not want to pass `--persist` explicitly. `--persist` wins when both are present. > > - Extensions and sandbox @@ -81,7 +81,7 @@ java -jar jawk-${project.version}-standalone.jar --list-ext - `--profile` is an executing mode. It keeps normal AWK output on stdout and writes the profiling report to stderr after execution finishes. - `--profile=` keeps normal AWK output on stdout and writes only the profiling report to the file. - `-S` affects compilation and execution, not just runtime behavior. -- `--posix` currently disables arrays-of-arrays syntax and related subarray-only operands, and stops treating gawk's `BEGINFILE` / `ENDFILE` patterns as special, in order to keep CLI compilation aligned with classic POSIX-style AWK expectations. +- `--posix` disables arrays-of-arrays syntax, related subarray-only operands, and all gawk `@` forms, and stops treating gawk's `BEGINFILE` / `ENDFILE` patterns as special, in order to keep CLI compilation aligned with classic POSIX-style AWK expectations. - `--posix` is rejected together with `-L`, because loading precompiled tuples bypasses source compilation entirely. - `-L` lets you skip source compilation, but the loaded tuples must still be compatible with the current runtime. - `-f` and `-L` are distinct paths: source files compile now, tuple files load now. diff --git a/src/site/markdown/cli.md b/src/site/markdown/cli.md index 80111db1..62312157 100644 --- a/src/site/markdown/cli.md +++ b/src/site/markdown/cli.md @@ -169,7 +169,7 @@ Use `-S` or `--sandbox` to switch to the sandboxed tuple compiler and runtime: $ java -jar jawk-${project.version}-standalone.jar -S -f script.awk input.txt ``` -Sandbox mode disables `system()`, input and output redirection, command pipelines, and related runtime features that are intentionally unsafe in a restricted host environment. +Sandbox mode disables `system()`, input and output redirection, command pipelines, `@include` source inclusion, and related features that are intentionally unsafe in a restricted host environment. ## See Also diff --git a/src/site/markdown/extensions.md b/src/site/markdown/extensions.md index 73108da2..747dd479 100644 --- a/src/site/markdown/extensions.md +++ b/src/site/markdown/extensions.md @@ -86,7 +86,25 @@ That keeps extension availability explicit and local to the embedding code. Beyond the extension functions, the interpreter itself implements gawk's `BEGINFILE` / `ENDFILE` special patterns, the `nextfile` statement, and the `ERRNO` and `ARGIND` special variables (see the [CLI guide](cli.html#BEGINFILE_and_ENDFILE_Rules)). Like the other gawk-specific syntax, `BEGINFILE` and `ENDFILE` are not special in POSIX mode. -Scripts that reference `SYMTAB` or `FUNCTAB` get honest, Jawk-shaped content, populated by the runtime itself (outside POSIX mode): `SYMTAB` holds the names of the program's globals, Jawk's special variables, and `-v`/host-supplied variables; `FUNCTAB` holds the names of the standard built-in functions (`split`, `substr`, ...), the program's user-defined functions, and the loaded extensions' function keywords. Command-line `name=value` operand assignments update `SYMTAB` live, as in gawk; ordinary in-script assignments are not reflected (the array is a startup snapshot, not gawk's live view). As in gawk, assigning a scalar to `SYMTAB` or `FUNCTAB` is a runtime error. +Jawk also supports gawk's source-level `@` syntax: + +```awk +@include "library.awk" +@namespace "report" + +function render(value) { return value } + +BEGIN { + callback = "report::render" + print @callback(42) +} +``` + +`@include` resolves relative paths from the including source, then searches `AWKPATH`, and includes each resolved file at most once. An included file cannot include a top-level program source. An included source begins in the `awk` namespace; the including source's namespace is restored afterward. `@namespace` qualifies variables and functions except identifiers made entirely of uppercase letters, while `awk::name` refers to the default namespace. Namespaced indirect calls require a fully qualified function name in the selector variable; an unqualified value refers to the default `awk` namespace. Indirect calls can dispatch user-defined, built-in, or loaded extension functions. Typed regexp literals use the related `@/re/` form. `@load` is recognized but intentionally reported as unsupported; load Java extensions with the CLI `-l` option or the Java API instead. + +All gawk `@` forms are rejected when POSIX mode is enabled. + +Scripts that reference `SYMTAB` or `FUNCTAB` get honest, Jawk-shaped content, populated by the runtime itself (outside POSIX mode): `SYMTAB` holds the names of the program's globals, Jawk's special variables, and `-v`/host-supplied variables; `FUNCTAB` holds the names of the standard built-in functions (`split`, `substr`, ...), the program's user-defined functions, and the loaded extensions' function keywords. Reads and writes through `SYMTAB` reflect declared globals and managed special variables live, but arbitrary elements cannot be added or deleted and writes must preserve each global's scalar or array type. `FUNCTAB` is read-only. As in gawk, assigning a scalar to `SYMTAB` or `FUNCTAB` is a runtime error. > [!NOTE] > Because these functions are registered by default, `gensub`, `typeof`, `isarray`, `asort`, `asorti`, `mkbool`, `patsplit`, `strtonum`, `systime`, `mktime`, `strftime`, `bindtextdomain`, `dcgettext`, and `dcngettext` become reserved function names. A script that uses them as variable or function identifiers must be run with an explicit extension list that omits `GawkExtension`. diff --git a/src/site/markdown/java-advanced.md b/src/site/markdown/java-advanced.md index d08cb969..3571f38f 100644 --- a/src/site/markdown/java-advanced.md +++ b/src/site/markdown/java-advanced.md @@ -100,6 +100,8 @@ awk.script(program) .execute(System.out); ``` +Sandboxed compilation rejects `@include` so an untrusted script cannot select and execute source files from the host filesystem. Pass every trusted source explicitly to the host API instead. + ## JSR 223 ScriptEngine Jawk also exposes a JSR 223 `ScriptEngine`: diff --git a/src/site/markdown/java.md b/src/site/markdown/java.md index a4000abc..4f562313 100644 --- a/src/site/markdown/java.md +++ b/src/site/markdown/java.md @@ -36,14 +36,14 @@ Awk awk = new Awk(settings); | `setLocale(Locale)` | `Locale.US` | Locale for numeric output formatting | | `setDefaultRS(String)` | Platform line separator | Default value for `RS`, the record separator | | `setUseSortedArrayKeys(boolean)` | `false` | Whether to keep associative array keys in sorted order | -| `setPosix(boolean)` | `false` | Enforce POSIX compile-time behavior, rejecting gawk syntax such as `a[i][j]`, `split(..., a[i])`, typed regexp literals (`@/re/`), and the `BEGINFILE` / `ENDFILE` special patterns | +| `setPosix(boolean)` | `false` | Enforce POSIX compile-time behavior, rejecting gawk syntax such as `a[i][j]`, `split(..., a[i])`, all `@` forms, and the `BEGINFILE` / `ENDFILE` special patterns | | `putVariable(String, Object)` | Empty map | Pre-set variables available before `BEGIN` | Output destination is specified per-call on the builder (`execute()`, `execute(PrintStream)`, `execute(OutputStream)`, `execute(Appendable)`, or `execute(AwkSink)`). See the [Custom Output](java-output.html) guide for details. For more on passing variables to scripts, see [Variables and Arguments](java-variables.html). -By default, Jawk accepts both classic multi-dimensional array syntax (`a[i, j]`) and gawk-style arrays of arrays (`a[i][j]`), and treats gawk's `BEGINFILE` / `ENDFILE` patterns as special. Enable POSIX mode when you need strict classic AWK parsing; it rejects arrays of arrays, subarray operands in array-only positions such as `split(..., a[i])`, `for (k in a[i])`, and `"x" in a[i]`, and typed regexp literals (`@/re/`), and it stops treating `BEGINFILE` / `ENDFILE` as special patterns: +By default, Jawk accepts both classic multi-dimensional array syntax (`a[i, j]`) and gawk-style arrays of arrays (`a[i][j]`), gawk's `@include`, `@namespace`, indirect-call, and typed-regexp syntax, and the `BEGINFILE` / `ENDFILE` patterns. Enable POSIX mode when you need strict classic AWK parsing; it rejects arrays of arrays, subarray operands in array-only positions such as `split(..., a[i])`, `for (k in a[i])`, and `"x" in a[i]`, rejects all `@` forms, and stops treating `BEGINFILE` / `ENDFILE` as special patterns: ```java AwkSettings settings = new AwkSettings(); diff --git a/src/test/java/io/jawk/AwkParserTest.java b/src/test/java/io/jawk/AwkParserTest.java index 89bc9b07..c0297152 100644 --- a/src/test/java/io/jawk/AwkParserTest.java +++ b/src/test/java/io/jawk/AwkParserTest.java @@ -204,6 +204,343 @@ public void testArraysOfArraysCanBeDisabled() { assertThrows(RuntimeException.class, () -> awk.compile("BEGIN { for (k in a[1]) print k }")); } + @Test + public void testIndirectFunctionCalls() throws Exception { + AwkTestSupport + .awkTest("Indirect calls dispatch user functions and builtins") + .script( + "function twice(value) { return value * 2 }\n" + + "BEGIN { user = \"twice\"; builtin = \"length\"; print @user(21), @builtin(\"jawk\") }") + .expectLines("42 4") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect length without arguments measures the current record") + .script("{ callback = \"length\"; print @callback() }") + .stdin("abcde\n") + .expectLines("5") + .runAndAssert(); + AwkTestSupport + .awkTest("Qualified variables can select indirect functions") + .script( + "@namespace \"ns\"\n" + + "function twice(value) { return value * 2 }\n" + + "BEGIN { callback = \"ns::twice\"; print @ns::callback(5) }") + .expectLines("10") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect srand converts a fractional string like direct srand") + .script( + "BEGIN { callback = \"srand\"; first = @callback(\"3.5\"); a = rand(); " + + "second = srand(\"3.5\"); b = rand(); print first, second, (a == b) }") + .expectLines("1 3 1") + .runAndAssert(); + AwkTestSupport + .awkTest("srand returns the previously requested seed") + .script("BEGIN { print srand(0), srand(5), srand(0) }") + .expectLines("1 0 5") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect split materializes an untyped array destination") + .script( + "BEGIN { callback = \"split\"; count = @callback(\"a b\", parts); " + + "print count, parts[1], parts[2] }") + .expectLines("2 a b") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect user calls preserve array parameters") + .script( + "function fill(values) { values[1] = 42; return values[1] }\n" + + "BEGIN { callback = \"fill\"; print @callback(parts), parts[1] }") + .expectLines("42 42") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect user calls materialize subarray parameters") + .script( + "function fill(values) { values[1] = 42 }\n" + + "BEGIN { callback = \"fill\"; @callback(parts[\"nested\"]); " + + "print parts[\"nested\"][1] }") + .expectLines("42") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect split materializes subarray parameters") + .script( + "BEGIN { callback = \"split\"; count = @callback(\"a b\", parts[\"nested\"]); " + + "print count, parts[\"nested\"][1], parts[\"nested\"][2] }") + .expectLines("2 a b") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect selectors are evaluated before arguments") + .script( + "function f(a, b) { print \"f\", a, b }\n" + + "function g(a, b) { print \"g\", a, b }\n" + + "BEGIN { callback = \"f\"; @callback(1, callback = \"g\") }") + .expectLines("f 1 g") + .runAndAssert(); + AwkTestSupport + .awkTest("Indirect scalar arguments retain their evaluation-time values") + .script( + "function f(a, b) { print a, b }\n" + + "BEGIN { x = 1; callback = \"f\"; @callback(x, x = 2) }") + .expectLines("1 2") + .runAndAssert(); + AwkTestSupport + .awkTest("Keywords cannot be literal indirect-call selectors") + .script("BEGIN { print @if(1) }") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Builtins cannot be literal indirect-call selectors") + .script("BEGIN { print @length(1) }") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Keywords cannot be qualified literal indirect-call selectors") + .script("BEGIN { print @ns::if(1) }") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Extension names remain valid qualified selector variables") + .script("BEGIN { ns::typeof = \"length\"; print @ns::typeof(\"abc\") }") + .expectLines("3") + .runAndAssert(); + } + + @Test + public void testNamespaces() throws Exception { + AwkTestSupport + .awkTest("Namespaces qualify functions and indirect call targets") + .script( + "@namespace \"lib\"\n" + + "function value() { return 42 }\n" + + "@namespace \"app\"\n" + + "function value() { return 7 }\n" + + "BEGIN { target = \"app::value\"; print lib::value(), @target() }") + .expectLines("42 7") + .runAndAssert(); + AwkTestSupport + .awkTest("Unqualified indirect targets stay in the awk namespace") + .script( + "@namespace \"app\"\n" + + "function value() { return 7 }\n" + + "BEGIN { target = \"value\"; print @target() }") + .expectThrow(RuntimeException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Only identifiers made entirely of uppercase letters stay in awk") + .script( + "@namespace \"ns\"\n" + + "BEGIN { MY_VAR = 1; F1 = 2; ABC = 3; " + + "print ns::MY_VAR, ns::F1, awk::ABC, awk::MY_VAR == \"\" }") + .expectLines("1 2 3 1") + .runAndAssert(); + AwkTestSupport + .awkTest("Standard builtins remain unqualified inside namespaces") + .script("@namespace \"ns\"\nBEGIN { print length(\"abc\"), int(3.5) }") + .expectLines("3 3") + .runAndAssert(); + AwkTestSupport + .awkTest("Builtin names cannot be redefined inside namespaces") + .script("@namespace \"ns\"\nfunction length(value) { return value }\n") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Builtin names cannot be used as namespaces") + .script("@namespace \"length\"\nBEGIN { print 1 }\n") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Extension names cannot be used as namespaces") + .script("@namespace \"typeof\"\nBEGIN { print 1 }\n") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Reserved words cannot follow namespace separators") + .script("BEGIN { ns::if = 3 }\n") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Builtin names cannot follow namespace separators") + .script("BEGIN { ns::length = 3 }\n") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("A parameter may follow an unrelated function with the same name") + .script( + "function awk::f() { return 1 }\n" + + "@namespace \"ns\"\n" + + "function g(f) { return f }\n" + + "BEGIN { print g(7), awk::f() }\n") + .expectLines("7 1") + .runAndAssert(); + AwkTestSupport + .awkTest("A parameter may precede an unrelated function with the same name") + .script( + "@namespace \"ns\"\n" + + "function g(f) { return f }\n" + + "function awk::f() { return 1 }\n" + + "BEGIN { print g(7), awk::f() }\n") + .expectLines("7 1") + .runAndAssert(); + AwkTestSupport + .awkTest("A parameter cannot match its own function name") + .script("function f(f) { return f }\n") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .cliTest("The awk namespace is accepted in command-line assignments") + .argument("-v", "awk::VALUE=42") + .script("BEGIN { print awk::VALUE }") + .expectLines("42") + .runAndAssert(); + AwkTestSupport + .cliTest("Qualified command-line operand assignments are accepted") + .file("input.txt", "record\n") + .script("@namespace \"ns\"\n{ print value, $0 }") + .operand("ns::value=42", "{{input.txt}}") + .expectLines("42 record") + .runAndAssert(); + AwkTestSupport + .cliTest("The awk namespace is accepted in operand assignments") + .file("input.txt", "record\n") + .script("@namespace \"ns\"\n{ print awk::value, $0 }") + .operand("awk::value=42", "{{input.txt}}") + .expectLines("42 record") + .runAndAssert(); + } + + @Test + public void testTernaryIdentifiersAdjacentToColon() throws Exception { + AwkTestSupport + .awkTest("A tight ternary can end its true branch with an identifier") + .script("BEGIN { x = 1; y = 2; print (x < y?x:y) }") + .expectLines("1") + .runAndAssert(); + AwkTestSupport + .awkTest("A qualified identifier can precede a ternary colon") + .script("@namespace \"ns\"\nBEGIN { cond = 1; x = 7; y = 9; print (cond?ns::x:y) }") + .expectLines("7") + .runAndAssert(); + AwkTestSupport + .awkTest("A tight ternary false branch can start with a parenthesis") + .script("BEGIN { x = 5; print (1?x:(2)) }") + .expectLines("5") + .runAndAssert(); + AwkTestSupport + .awkTest("A qualified tight ternary false branch can start with a parenthesis") + .script("@namespace \"ns\"\nBEGIN { x = 7; print (1?ns::x:(9)) }") + .expectLines("7") + .runAndAssert(); + } + + @Test + public void testIncludeDirective() throws Exception { + AwkTestSupport + .cliTest("Include resolves relative to the main source and restores its namespace") + .file( + "main.awk", + "@namespace \"main\"\n" + + "@include \"empty.awk\"\n" + + "@include \"lib.awk\"\n" + + "function answer() { return 7 }\n" + + "BEGIN { print lib::answer(), answer() }\n") + .file( + "lib.awk", + "@namespace \"lib\"\n" + + "function answer() { return 42 }\n") + .file("empty.awk", "") + .argument("-f", "{{main.awk}}") + .expectLines("42 7") + .runAndAssert(); + AwkTestSupport + .cliTest("An include without a final newline restores its parent's namespace") + .file( + "main.awk", + "@namespace \"main\"\n" + + "@include \"lib.awk\"\n" + + "function answer() { return 7 }\n" + + "BEGIN { print main::answer() }\n") + .file("lib.awk", "@namespace \"lib\"") + .argument("-f", "{{main.awk}}") + .expectLines("7") + .runAndAssert(); + AwkTestSupport + .cliTest("A top-level source cannot include itself") + .file("main.awk", "@include \"main.awk\"\nBEGIN { print 1 }\n") + .argument("-f", "{{main.awk}}") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .cliTest("An include cycle cannot include its top-level source") + .file("main.awk", "@include \"lib.awk\"\nBEGIN { print 1 }\n") + .file("lib.awk", "@include \"main.awk\"\nBEGIN { print 2 }\n") + .argument("-f", "{{main.awk}}") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .cliTest("An include directive requires a statement terminator") + .file("main.awk", "@include \"lib.awk\" BEGIN { print 1 }\n") + .file("lib.awk", "BEGIN { print 2 }\n") + .argument("-f", "{{main.awk}}") + .expectThrow(ParserException.class) + .runAndAssert(); + AwkTestSupport + .cliTest("A semicolon terminates an include directive") + .file("main.awk", "@include \"lib.awk\"; BEGIN { print 1 }\n") + .file("lib.awk", "BEGIN { print 2 }\n") + .argument("-f", "{{main.awk}}") + .expectLines("2", "1") + .runAndAssert(); + } + + @Test + public void testIncludeCanonicalizesSymlinkAliases() throws Exception { + AwkTestSupport + .cliTest("Symlink aliases include the underlying file only once") + .file( + "main.awk", + "@include \"lib.awk\"\n" + + "@include \"alias.awk\"\n") + .file("lib.awk", "BEGIN { print \"included\" }\n") + .symlink("alias.awk", "lib.awk") + .argument("-f", "{{main.awk}}") + .expectLines("included") + .runAndAssert(); + } + + @Test + public void testAtSyntaxCanBeDisabled() throws Exception { + AwkSettings settings = new AwkSettings(); + settings.setPosix(true); + + AwkTestSupport + .awkTest("gawk @ syntax is unavailable in POSIX mode") + .withAwk(new Awk(settings)) + .script("@namespace \"example\"\nBEGIN { print 1 }") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("gawk namespace-qualified names are unavailable in POSIX mode") + .withAwk(new Awk(settings)) + .script("BEGIN { example::value = 1 }") + .expectThrow(LexerException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("Ternary colons remain available in POSIX mode") + .withAwk(new Awk(settings)) + .script("BEGIN { yes = 1; no = 2; print (1?yes:no) }") + .expectLines("1") + .runAndAssert(); + } + + @Test + public void testUnsupportedAtDirective() throws Exception { + AwkTestSupport + .awkTest("Unsupported gawk @ directives fail intentionally") + .script("@load \"example\"\nBEGIN { print 1 }") + .expectThrow(ParserException.class) + .runAndAssert(); + } + @Test public void testOperatorPrecedence() throws Exception { AwkTestSupport diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 88686f90..fdc5c52b 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -1393,6 +1393,17 @@ public void sandboxRejectsInputRedirectionDuringCompile() { () -> awk.compile("BEGIN { getline x < \"file\" }")); } + @Test + public void sandboxRejectsSourceInclusionDuringCompile() throws Exception { + AwkTestSupport + .awkTest("sandbox rejects source inclusion") + .withAwk(new SandboxedAwk()) + .file("included.awk", "BEGIN { print \"included\" }\n") + .script("@include \"{{included.awk}}\"\nBEGIN { print \"main\" }") + .expectThrow(AwkSandboxException.class) + .runAndAssert(); + } + @Test public void sandboxRejectsRedirectionAtRuntime() throws Exception { AwkProgram program = AWK.compile("BEGIN { print \"hi\" > \"sandbox_out.txt\" } "); diff --git a/src/test/java/io/jawk/AwkTestSupport.java b/src/test/java/io/jawk/AwkTestSupport.java index dd30bf65..1b257179 100644 --- a/src/test/java/io/jawk/AwkTestSupport.java +++ b/src/test/java/io/jawk/AwkTestSupport.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeNoException; import static org.junit.Assume.assumeTrue; import java.io.BufferedReader; @@ -500,6 +501,7 @@ public AwkTestBuilder withInputSource(InputSource inputSourceParam) { protected AwkTestCase buildTestCase( TestLayout layout, Map files, + Map symlinks, List operands, List placeholders) { if (useTempDir && !preAssignments.containsKey("TEMPDIR")) { @@ -508,6 +510,7 @@ protected AwkTestCase buildTestCase( return new AwkTestCase( layout, files, + symlinks, operands, placeholders, requiresPosix, @@ -620,6 +623,7 @@ public CliTestBuilder env(Map values) { protected CliTestCase buildTestCase( TestLayout layout, Map files, + Map symlinks, List operands, List placeholders) { if (useTempDir && !assignments.containsKey("TEMPDIR")) { @@ -628,6 +632,7 @@ protected CliTestCase buildTestCase( return new CliTestCase( layout, files, + symlinks, operands, placeholders, requiresPosix, @@ -651,6 +656,7 @@ private abstract static class BaseTestBuilder> { protected String script; protected String stdin; protected final Map fileContents = new LinkedHashMap<>(); + protected final Map symbolicLinks = new LinkedHashMap<>(); protected final List operandSpecs = new ArrayList<>(); protected final List pathPlaceholders = new ArrayList<>(); protected String expectedOutput; @@ -706,6 +712,22 @@ public B file(String name, String contents) { return (B) this; } + /** + * Adds a symbolic link inside the per-test temporary directory. The target + * is resolved relative to that directory and should normally name a file + * configured through {@link #file(String, String)}. The test is skipped + * when the platform does not permit symbolic-link creation. + * + * @param name relative path of the symbolic link + * @param target relative path of its target + * @return this builder for method chaining + */ + @SuppressWarnings("unchecked") + public B symlink(String name, String target) { + symbolicLinks.put(name, target); + return (B) this; + } + /** * Adds operands to pass to the script when it is executed. Placeholders * are resolved at runtime. @@ -876,9 +898,10 @@ public ConfiguredTest build() { expectedExitCode, expectedException); Map files = new LinkedHashMap<>(fileContents); + Map symlinks = new LinkedHashMap<>(symbolicLinks); List operands = new ArrayList<>(operandSpecs); List placeholders = new ArrayList<>(pathPlaceholders); - return buildTestCase(layout, files, operands, placeholders); + return buildTestCase(layout, files, symlinks, operands, placeholders); } /** @@ -905,6 +928,7 @@ public void runAndAssert() throws Exception { protected abstract BaseTestCase buildTestCase( TestLayout layout, Map fileContents, + Map symbolicLinks, List operandSpecs, List pathPlaceholders); } @@ -912,6 +936,7 @@ protected abstract BaseTestCase buildTestCase( private abstract static class BaseTestCase implements ConfiguredTest { private final TestLayout layout; private final Map fileContents; + private final Map symbolicLinks; private final List operandSpecs; private final List pathPlaceholders; private final boolean requiresPosix; @@ -919,11 +944,13 @@ private abstract static class BaseTestCase implements ConfiguredTest { BaseTestCase( TestLayout layout, Map fileContents, + Map symbolicLinks, List operandSpecs, List pathPlaceholders, boolean requiresPosix) { this.layout = layout; this.fileContents = fileContents; + this.symbolicLinks = symbolicLinks; this.operandSpecs = operandSpecs; this.pathPlaceholders = pathPlaceholders; this.requiresPosix = requiresPosix; @@ -1024,6 +1051,20 @@ protected ExecutionEnvironment prepareEnvironment() throws IOException { } placeholders.put(entry.getKey(), path); } + for (Map.Entry entry : symbolicLinks.entrySet()) { + Path link = tempDir.resolve(entry.getKey()); + Path parent = link.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try { + Files.createSymbolicLink(link, tempDir.resolve(entry.getValue())); + } catch (IOException | UnsupportedOperationException | SecurityException ex) { + deleteRecursively(tempDir); + assumeNoException("Symbolic links are unavailable for " + layout.description, ex); + } + placeholders.put(entry.getKey(), link); + } for (String placeholder : pathPlaceholders) { Path path = tempDir.resolve(placeholder); Path parent = path.getParent(); @@ -1062,6 +1103,7 @@ private static final class AwkTestCase extends BaseTestCase { AwkTestCase( TestLayout layout, Map fileContents, + Map symbolicLinks, List operandSpecs, List pathPlaceholders, boolean requiresPosix, @@ -1071,7 +1113,7 @@ private static final class AwkTestCase extends BaseTestCase { InputSource inputSource, Reader scriptReader, Path scriptPath) { - super(layout, fileContents, operandSpecs, pathPlaceholders, requiresPosix); + super(layout, fileContents, symbolicLinks, operandSpecs, pathPlaceholders, requiresPosix); this.preAssignments = new LinkedHashMap<>(preAssignments); this.customAwk = customAwk; this.extensions = new ArrayList<>(extensions); @@ -1137,6 +1179,7 @@ private static final class CliTestCase extends BaseTestCase { CliTestCase( TestLayout layout, Map fileContents, + Map symbolicLinks, List operandSpecs, List pathPlaceholders, boolean requiresPosix, @@ -1145,7 +1188,7 @@ private static final class CliTestCase extends BaseTestCase { Map environment, boolean redirectErrorStream, InputStream stdinStream) { - super(layout, fileContents, operandSpecs, pathPlaceholders, requiresPosix); + super(layout, fileContents, symbolicLinks, operandSpecs, pathPlaceholders, requiresPosix); this.argumentSpecs = new ArrayList<>(argumentSpecs); this.assignments = new LinkedHashMap<>(assignments); this.environment = new LinkedHashMap<>(environment); diff --git a/src/test/java/io/jawk/CliOptionTest.java b/src/test/java/io/jawk/CliOptionTest.java index ed8a9190..9061f856 100644 --- a/src/test/java/io/jawk/CliOptionTest.java +++ b/src/test/java/io/jawk/CliOptionTest.java @@ -99,6 +99,25 @@ public void profileOptionRecordsFunctionThatExitsWithoutReturning() throws Excep assertTrue(result.errorOutput().contains("stop")); } + @Test + public void profileOptionRecordsIndirectUserFunctions() throws Exception { + AwkTestSupport.TestResult result = AwkTestSupport + .cliTest("CLI --profile records indirect user functions") + .argument("--profile") + .script( + "function inner() { return 1 } " + + "function outer() { callback = \"inner\"; extension = \"typeof\"; " + + "@extension(1); return @callback() } " + + "BEGIN { print outer() }") + .expectLines("1") + .run(); + + result.assertExpected(); + assertTrue(result.errorOutput().contains("inner")); + assertTrue(result.errorOutput().contains("outer")); + assertTrue(result.errorOutput().contains("typeof")); + } + @Test public void profileOptionWithFilenameWritesReportToFile() throws Exception { File profile = tempFolder.newFile("profile.txt"); @@ -399,6 +418,26 @@ public void persistOptionPersistsAssociativeArraysAcrossCliRuns() throws Excepti .runAndAssert(); } + @Test + public void persistOptionExcludesRuntimeMetaTables() throws Exception { + File memory = new File(tempFolder.getRoot(), "symtab-memory.bin"); + + AwkTestSupport + .cliTest("CLI persist excludes SYMTAB from its serialized snapshot") + .argument("--persist", memory.getAbsolutePath()) + .script("BEGIN { value = 7; print SYMTAB[\"value\"] }") + .expectLines("7") + .expectExit(0) + .runAndAssert(); + AwkTestSupport + .cliTest("CLI persist restores user globals and rebuilds SYMTAB") + .argument("--persist", memory.getAbsolutePath()) + .script("BEGIN { print value, SYMTAB[\"value\"] }") + .expectLines("7 7") + .expectExit(0) + .runAndAssert(); + } + @Test public void persistentMemoryEnvironmentVariablePersistsUserGlobalsAcrossCliRuns() throws Exception { File memory = new File(tempFolder.getRoot(), "env-memory.bin"); diff --git a/src/test/java/io/jawk/ExtensionTest.java b/src/test/java/io/jawk/ExtensionTest.java index 765da2c0..42458152 100644 --- a/src/test/java/io/jawk/ExtensionTest.java +++ b/src/test/java/io/jawk/ExtensionTest.java @@ -55,6 +55,18 @@ public void testExtension() throws Exception { .runAndAssert(); } + @Test + public void testIndirectExtension() throws Exception { + AwkTestSupport + .awkTest("indirect extension invocation") + .script( + "BEGIN { ab[1] = \"a\"; ab[2] = \"b\"; " + + "callback = \"myExtensionFunction\"; print @callback(2, ab) }") + .withExtensions(new TestExtension()) + .expectLines("abab") + .runAndAssert(); + } + @Test public void testAnnotatedExtensionArgumentValidation() throws Exception { AwkTestSupport diff --git a/src/test/java/io/jawk/GawkExtensionTest.java b/src/test/java/io/jawk/GawkExtensionTest.java index 9bfda074..9d449d6e 100644 --- a/src/test/java/io/jawk/GawkExtensionTest.java +++ b/src/test/java/io/jawk/GawkExtensionTest.java @@ -382,6 +382,103 @@ public void symtabSeesRuntimeManagedGlobals() throws Exception { .runAndAssert(); } + @Test + public void symtabReadsAndWritesRuntimeValuesLive() throws Exception { + AwkTestSupport + .awkTest("SYMTAB reads globals live and writes through to them") + .script( + "BEGIN { x = 1; before = SYMTAB[\"x\"]; x = 2; " + + "after = SYMTAB[\"x\"]; SYMTAB[\"x\"] = 7; print before, after, x }") + .expectLines("1 2 7") + .runAndAssert(); + AwkTestSupport + .awkTest("Reading an unknown SYMTAB element returns an empty value") + .script("BEGIN { print \"[\" SYMTAB[\"xyzzy\"] \"]\" }") + .expectLines("[]") + .runAndAssert(); + AwkTestSupport + .awkTest("SYMTAB rejects assignments to arbitrary elements") + .script("BEGIN { SYMTAB[\"xyzzy\"] = 5 }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void symtabCannotReplaceArrayGlobalsWithScalars() throws Exception { + AwkTestSupport + .awkTest("SYMTAB preserves array global types") + .script("BEGIN { values[1] = 1; SYMTAB[\"values\"] = 3 }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void symtabCannotReplaceScalarGlobalsWithArrays() throws Exception { + AwkTestSupport + .awkTest("SYMTAB preserves scalar global types") + .script("BEGIN { if (0) value = 1; SYMTAB[\"value\"][1] = 3 }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void symtabRoundTripsSpecialVariables() throws Exception { + AwkTestSupport + .awkTest("SYMTAB reads and writes managed special variables") + .script( + "BEGIN { SYMTAB[\"NR\"] = 9; SYMTAB[\"OFS\"] = \":\"; " + + "print NR, SYMTAB[\"NR\"]; print 1, 2; print SYMTAB[\"OFS\"] }") + .expectLines("9:9", "1:2", ":") + .runAndAssert(); + AwkTestSupport + .awkTest("SYMTAB writes ARGC without recursive assignment") + .script("BEGIN { SYMTAB[\"ARGC\"] = 4; print ARGC, SYMTAB[\"ARGC\"] }") + .expectLines("4 4") + .runAndAssert(); + AwkTestSupport + .awkTest("SYMTAB writes match result variables") + .script( + "BEGIN { SYMTAB[\"RSTART\"] = 7; SYMTAB[\"RLENGTH\"] = 3; " + + "print RSTART, SYMTAB[\"RSTART\"], RLENGTH, SYMTAB[\"RLENGTH\"] }") + .expectLines("7 7 3 3") + .runAndAssert(); + } + + @Test + public void symtabIterationUsesItsMaterializedKeysAndLiveValues() throws Exception { + AwkTestSupport + .awkTest("SYMTAB iteration exposes declared globals with live values") + .script( + "BEGIN { x = 1; x = 7; " + + "for (key in SYMTAB) if (key == \"x\") { seen++; value = SYMTAB[key] } " + + "print seen, value }") + .expectLines("1 7") + .runAndAssert(); + } + + @Test + public void symtabIterationHonorsSortedArrayKeys() throws Exception { + AwkTestSupport + .cliTest("SYMTAB uses sorted iteration when -t is enabled") + .argument("-t") + .script( + "BEGIN { a = 1; aa = 2; " + + "for (key in SYMTAB) if (key == \"a\" || key == \"aa\") print key }") + .expectLines("a", "aa") + .runAndAssert(); + } + + @Test + public void symtabEntrySetExposesLiveValues() throws Exception { + AwkTestSupport + .awkTest("SYMTAB value sorting uses live global values") + .script( + "BEGIN { a = 2; z = 1; PROCINFO[\"sorted_in\"] = \"@val_num_asc\"; " + + "for (key in SYMTAB) if (key == \"a\" || key == \"z\") print key }") + .expectLines("z", "a") + .runAndAssert(); + } + @Test public void ignoreCaseAppliesToComparisonsAndIndex() throws Exception { // gawk's IGNORECASE covers string relational operators and index(), @@ -419,6 +516,34 @@ public void metaTablesCannotBeUsedAsScalars() throws Exception { .runAndAssert(); } + @Test + public void symtabCannotBeDeleted() throws Exception { + AwkTestSupport + .awkTest("SYMTAB elements cannot be deleted") + .script("BEGIN { value = 1; delete SYMTAB[\"value\"] }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("SYMTAB cannot be cleared") + .script("BEGIN { delete SYMTAB }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void functabIsReadOnly() throws Exception { + AwkTestSupport + .awkTest("FUNCTAB elements cannot be assigned") + .script("BEGIN { FUNCTAB[\"length\"] = \"replacement\" }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("FUNCTAB elements cannot be deleted") + .script("BEGIN { delete FUNCTAB[\"length\"] }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void perRunVariablesAreVisibleToSymtabAndExtensions() throws Exception { // Per-run overrides without a compiled slot must be observable through @@ -516,6 +641,18 @@ public void extraFunctionArgumentWarningGoesToErrorStreamNotOutput() throws Exce assertTrue( "extra-argument warning should be printed to the error stream", result.errorOutput().contains("called with more arguments than declared")); + + AwkTestSupport.TestResult indirectResult = AwkTestSupport + .cliTest("indirect extra-argument warning includes source and line") + .script("function f(a) { return a } BEGIN { name = \"f\"; print @name(1, 2) }") + .expect("1\n") + .run(); + indirectResult.assertExpected(); + assertTrue( + "indirect warning should include a source line: " + indirectResult.errorOutput(), + indirectResult + .errorOutput() + .matches("(?s).*gawk: .+:\\d+: warning: function `f'.*")); } @Test @@ -703,6 +840,15 @@ public void typeofReturnsGawkCategories() throws Exception { .runAndAssert(); } + @Test + public void indirectTypeofPreservesUntypedVariables() throws Exception { + AwkTestSupport + .awkTest("indirect typeof receives an untyped variable") + .script("BEGIN { callback = \"typeof\"; print @callback(unset), typeof(unset) }") + .expectLines("untyped untyped") + .runAndAssert(); + } + @Test public void typeofReportsStrnumForNumericInputFields() throws Exception { AwkTestSupport