Fixes #37 Performance updates

- more interface changes
  - ITransductor.Metrics inner class refactored
  - comments only touched in IParameterizedEffector
- merged nil/paste loop with sum/product/scan traps in Transductor.run()
- add thresholds for injecting msum/mproduct instrumentation in compiler
  - add definable runtime properties to enable per model thresholds
    - ribose.product.threshold (default = 12,  was hardcoded = 3)
    - ribose.sum.threshold (default = 128, was hardcoded = 64)
- drop per run ms stats for file runner and test runner tests
- add nil-paste/sum/product/scan run and run length counters to metrics
  - report average run length and percentage input trapped per trap

Signed-off-by: jrte <jrte.project@gmail.com>

build.xml

@@ -48,6 +48,7 @@
 	<property name="regex.linuxkernel" value="([JFMASOND][a-z]+ [0-9]+ (?:[0-9]+:)+[0-9]+) ([-.:A-Za-z_0-9]*) kernel: \[[ ]*[0-9]+\.[0-9]+\] (DROPPED|ABORTED|LIMITED) IN=([-.:A-Za-z_0-9]*) OUT=([-.:A-Za-z_0-9]*)(?: MAC=([-.:A-Za-z_0-9]*))? SRC=([-.:A-Za-z_0-9]*) DST=([-.:A-Za-z_0-9]*).* PROTO=([-.:A-Za-z_0-9]*)(?:.* SPT=([-.:A-Za-z_0-9]*) DPT=([-.:A-Za-z_0-9]*))?"/>
 	<property name="ribose.inbuffer.size" value="65536"/>
 	<property name="ribose.outbuffer.size" value="65536"/>
+	<property name="jargs.ribose.compile" value="-Dribose.product.threshold=12 -Dribose.sum.threshold=128"/>
 	<property name="jargs.jit.diagnostic" value="-XX:+UnlockDiagnosticVMOptions -XX:+PrintCompilation -XX:+PrintInlining"/>
 	<property name="jargs.jit.inline" value="-XX:CompileCommandFile=etc/jit/ribose.jit"/>
 	<property name="jargs.gc.buffers" value="-Dribose.inbuffer.size=${ribose.inbuffer.size} -Dribose.outbuffer.size=${ribose.outbuffer.size}"/>
@@ -293,7 +294,7 @@
 		<delete file="build/Test.map"/>
 		<echo message="Building build/Test.model"/>
 		<exec failonerror="true" executable="bash">
-			<arg line="./ribose ${jargs} compile --target-path ${jars.dir}/${ribose.version}-test.jar --target com.characterforming.jrte.test.TestTarget ${build.automata} build/Test.model"/>
+			<arg line="./ribose ${jargs} ${jargs.ribose.compile} compile --target-path ${jars.dir}/${ribose.version}-test.jar --target com.characterforming.jrte.test.TestTarget ${build.automata} build/Test.model"/>
 		</exec>
 		<fixcrlf file="build/Test.map" eol="lf"/>
 	</target>

etc/sh/bench

@@ -15,6 +15,15 @@ jrte=$(realpath $(dirname $0)/../..)
 jars=$jrte/jars
 version=$(cat $jrte/version)
 path="$jars/$version.jar:$jars/$version-test.jar"
+vmargs=""
+while [[ ! "$1" =~ ^(--tsv|[0-9]+) ]]; do
+	vmargs="$vmargs $1";
+	shift
+	if (($#==0)); then
+		execHelp
+		exit 0
+	fi
+done
 format=cat
 if [[ "$1" == "--tsv" ]]; then	
 	format=strip
@@ -32,5 +41,5 @@ input=$4
 shift 4
 output="-Djrte.out.enabled=false -Dregex.out.enabled=false"
 for i in $(seq $n); do
-	${JAVA_HOME}/bin/java -Dfile.encoding=UTF-8 $output -cp "$path" com.characterforming.jrte.test.FileRunner --nil "$model" $transducer "$input" "$@"|$format
+	${JAVA_HOME}/bin/java $vmargs -Dfile.encoding=UTF-8 $output -cp "$path" com.characterforming.jrte.test.FileRunner --nil "$model" $transducer "$input" "$@"|$format
 done

patterns/ribose/Automaton.inr

@@ -1,6 +1,6 @@
 Sign = ('-', paste)?;
 Number = (digit, paste)+;
-Symbol = [count[`~length` `!eol`]] (byte, paste count)* eol;
+Symbol = (byte, paste count)* eol;
 Field = select[X] clear;
 Eol = cr? nl;
 Inr = 'INR';
@@ -17,7 +17,7 @@ Automaton = nil? (
 		(tab, Field@(X,`~to`)) Number
 		(tab, Field@(X,`~tape`)) Sign? Number
 		(tab, Field@(X,`~length`)) Number
-		(tab, Field@(X,`~symbol`)) Symbol
+		(tab, (Field@(X,`~symbol`)) count[`~length` `!eol`]) Symbol
 		(Eol, transition (Field@(X,`~from`)))
 	)*
 	(eos, automaton stop)

src/com/characterforming/jrte/engine/ModelCompiler.java

@@ -61,7 +61,10 @@
 public final class ModelCompiler extends Model implements ITarget, AutoCloseable {
 
 	private static final long VERSION = 210;
+	private static final String AUTOMATON = "Automaton";
 	private static final String AMBIGUOUS_STATE_MESSAGE = "%1$s: Ambiguous state %2$d";
+	private static final int MIN_PRODUCT_LENGTH = Integer.parseInt(System.getProperty("ribose.product.threshold", "12"));
+	private static final int MIN_SUM_SIZE = Integer.parseInt(System.getProperty("ribose.sum.threshold", "128"));
 
 	private Bytes transducerName;
 	private int transducerOrdinal;
@@ -78,7 +81,6 @@ public final class ModelCompiler extends Model implements ITarget, AutoCloseable
 	private int[][][] kernelMatrix;
 	private int transition;
 
-	private static final String AUTOMATON = "Automaton";
 
 	record Transition (int from, int to, int tape, Bytes symbol, boolean isFinal) {}
 
@@ -683,7 +685,7 @@ private void factor(final int[][][] transitionMatrix) throws CharacterCodingExce
 						break;
 					}
 				}
-			} else if (selfCount > 64) {
+			} else if (selfCount > ModelCompiler.MIN_SUM_SIZE) {
 				assert msumStateEffects[state] == null;
 				Argument argument = new Argument(-1, new BytesArray(new byte[][] { Arrays.copyOf(selfBytes, selfCount) }));
 				int msumParameterIndex = super.compileParameters(msumOrdinal, argument);
@@ -748,7 +750,7 @@ private void factor(final int[][][] transitionMatrix) throws CharacterCodingExce
 					assert inputEquivalenceToken[exitEquivalent[toState]] >= 0;
 					assert inputEquivalenceToken[exitEquivalent[toState]] < nulSignal;
 					int nextState = this.walk(toState, walkedBytes, walkResult, exitEquivalent, inputEquivalenceToken);
-					if (walkResult[0] > 3) {
+					if (walkResult[0] > ModelCompiler.MIN_PRODUCT_LENGTH) {
 						assert this.inputEquivalenceIndex[walkedBytes[0]] == exitEquivalent[toState];
 						if (mproductStateEffects[toState] == null) {
 							byte[][] product = new byte[][] { Arrays.copyOfRange(walkedBytes, 0, walkResult[0]) };

src/com/characterforming/jrte/engine/Transductor.java

@@ -249,7 +249,7 @@ public long asInteger(int fieldOrdinal) throws EffectorException {
 					n = (10 * n) + (digit - 48);
 				} else {
 					throw new NumberFormatException(String.format(
-							"Not a numeric value '%1$s'", this.toString()));
+						"Not a numeric value '%1$s'", this.toString()));
 				}
 			}
 			return sign * n;
@@ -276,7 +276,7 @@ public double asReal(int fieldOrdinal) throws EffectorException {
 					mark = true;
 				} else {
 					throw new NumberFormatException(String.format(
-							"Not a floating point value '%1$s'", this.toString()));
+						"Not a floating point value '%1$s'", this.toString()));
 				}
 			}
 			return f * n;
@@ -328,9 +328,8 @@ public Status status() {
 			} else {
 				return this.inputStack.isEmpty() ? Status.PAUSED : Status.RUNNABLE;
 			}
-		} else {
-			return Status.PROXY;
 		}
+		return Status.PROXY;
 	}
 
 	@Override // @see com.characterforming.ribose.ITransductor#output(OutputStream)
@@ -398,6 +397,7 @@ public ITransductor run() throws EffectorException, DomainErrorException {
 		int token = -1, state = 0, last = -1, signal = 0;
 		if (this.prologue != null) {
 			signal = this.prologue.signal();
+			this.matchMode = MATCH_NONE;
 			this.prologue = null;
 		}
 		Input input = Input.empty;
@@ -412,8 +412,10 @@ public ITransductor run() throws EffectorException, DomainErrorException {
 				this.selected = this.transducer.selected;
 				state = this.transducer.state;
 I:			do {
+					int pos = input.position;
 					// get next input token
 					if (signal > 0) {
+						pos = input.position;
 						token = signal;
 						signal = 0;
 					} else {
@@ -424,49 +426,60 @@ public ITransductor run() throws EffectorException, DomainErrorException {
 								break T;
 							}
 						}
-						token = 0xff & input.array[input.position++];
+						pos = input.position++;
+						token = 0xff & input.array[pos];
 					}
 
-					// absorb self-referencing (msum,mscan) or sequential (mproduct) transitions with nil effect
-					while (this.matchMode != MATCH_NONE && token < SIGNUL) {
+					int action = NIL, mode = this.matchMode;
+S:				do {
 						switch (this.matchMode) {
+						// trap runs in (nil* paste*)* effector space
+						case MATCH_NONE:
+							do {
+								long transition = transitionMatrix[state + inputFilter[token]];
+								last = state; state = Transducer.state(transition);
+								action = Transducer.action(transition);
+								if (action == PASTE)
+									this.value.paste((byte)token);
+								else if (action != NIL)
+									break S;
+								token = input.position < input.limit
+								? 0xff & input.array[input.position++]
+								: -1;
+							} while (token >= 0);
+							break;
+						// absorb self-referencing (msum,mscan) or sequential (mproduct) transitions with nil effect
 						case MATCH_SUM:
-							token = sumTrap(input, token);
+							if (token < SIGNUL)
+								token = sumTrap(input, token);
 							break;
 						case MATCH_PRODUCT:
-							token = productTrap(input, token);
+							if (token < SIGNUL)
+								token = productTrap(input, token);
 							break;
 						case MATCH_SCAN:
-							token = scanTrap(input, token);
+							if (token < SIGNUL)
+								token = scanTrap(input, token);
 							break;
 						default:
 							assert false;
 							break;
 						}
 						if (token < 0) {
-							continue I;
-						}
-					}
-
-					// trap runs in (nil* paste*)* effector space
-					int action;
-S:				do {
-						long transition = transitionMatrix[state + inputFilter[token]];
-						last = state; state = Transducer.state(transition);
-						action = Transducer.action(transition);
-						if (action == PASTE) {
-							this.value.paste((byte)token);
-						} else if (action != NIL) {
-							break S;
-						}
-						if (input.position < input.limit) {
-							token = 0xff & input.array[input.position++];
-						} else {
+							if (input.position > pos) {
+								this.metrics.traps[mode][0] += input.position > pos ? 1 : 0;
+								this.metrics.traps[mode][1] += input.position - pos;
+							}
 							continue I;
 						}
 					} while (true);
+					if (input.position > pos) {
+						this.metrics.traps[mode][0] += input.position > pos ? 1 : 0;
+						this.metrics.traps[mode][1] += input.position - pos;
+					}
 
 					// effect action and check for transducer or input stack adjustment
+					assert this.matchMode == MATCH_NONE;
 					int aftereffects = effect(action, token, effectorVector);
 					if (aftereffects != IEffector.RTX_NONE) {
 						if (0 != (aftereffects & IEffector.RTX_INPUT)) {
@@ -624,17 +637,14 @@ private int effect(int action, int token, int[] effectorVector)
 	}
 
 	private int sumTrap(Input input, int token) {
-		final int anchor = input.position;
 		final long[] matchMask = this.matchSum;
 		while (0 != (matchMask[token >> 6] & (1L << (token & 0x3f)))) {
 			if (input.position < input.limit) {
 				token = 0xff & input.array[input.position++];
 			} else {
-				this.metrics.sum += (input.position - anchor);
 				return -1;
 			}
 		}
-		this.metrics.sum += (input.position - anchor);
 		this.matchMode = MATCH_NONE;
 		return token;
 	}
@@ -656,36 +666,31 @@ private int productTrap(Input input, int token) {
 				}
 			} else {
 				this.errorInput = 0xff & match;
-				this.metrics.product += mpos;
 				this.matchMode = MATCH_NONE;
 				return SIGNUL;
 			}
 		}
-		this.metrics.product += mpos;
 		this.matchMode = MATCH_NONE;
 		return 0xff & match;
 	}
 
 	private int scanTrap(Input input, int token) {
-		final int anchor = input.position;
 		final int matchToken = this.matchByte;
 		while (token != matchToken) {
 			if (input.position < input.limit) {
 				token = 0xff & input.array[input.position++];
 			} else {
-				this.metrics.scan += (input.position - anchor);
 				return -1;
 			}
 		}
-		this.metrics.scan += (input.position - anchor);
 		this.matchMode = MATCH_NONE;
 		return token;
 	}
 
 	private String getErrorInput(int last, int state) {
 		TransducerState top = this.transducerStack.peek();
 		int eqCount = top.get().getInputEquivalentsCount();
-		state /= eqCount; last /= eqCount;
+		top.state = state; state /= eqCount; last /= eqCount;
 		StringBuilder message = new StringBuilder(256);
 		message.append(String.format("Domain error on (%1$d~%2$d) in %3$s [%4$d]->[%5$d]%n,\tTransducer stack:%n",
 			this.errorInput, this.errorInput >= 0 ? top.get().getInputFilter()[this.errorInput] : this.errorInput, 
@@ -900,7 +905,7 @@ public Integer[] allocateParameters(int parameterCount) {
 		@Override
 		public Integer compileParameter(final IToken[] parameterList) throws TargetBindingException {
 			if (parameterList.length != 1) {
-				throw new TargetBindingException("The signal effector accepts at most one parameter");
+				throw new TargetBindingException("The signal effector accepts exactly one parameter");
 			} else if (parameterList[0] instanceof Token token) {
 				if (token.getType() == IToken.Type.SIGNAL) {
 					int ordinal = token.getOrdinal();
@@ -922,7 +927,7 @@ public Integer compileParameter(final IToken[] parameterList) throws TargetBindi
 				}
 			} else {
 				throw new TargetBindingException(String.format("Unknown IToken implementation class '%1$s'",
-				parameterList[0].getClass().getTypeName()));
+					parameterList[0].getClass().getTypeName()));
 			}
 		}
 	}

src/com/characterforming/jrte/test/FileRunner.java

@@ -117,20 +117,30 @@ public static void main(final String[] args) {
 								}
 								t1 = System.nanoTime() - t0;
 								if (i >= 10) {
-									System.out.print(String.format("%4d", t1/(1000000)));
 									tjrte += t1;
 								}
 								assert !trex.status().isRunnable();
 								trex.stop();
 								assert trex.status().isStopped();
 							}
 						}
-						double mbps = (tjrte > 0) ? (double)(metrics.bytes*1000000000l) / (double)(tjrte*1024*1024) : -1;
-						double mps = (metrics.bytes > 0) ? ((double)(100 * metrics.sum) / (double)metrics.bytes) : -1;
-						double mpr = (metrics.bytes > 0) ? ((double)(100 * metrics.product) / (double)metrics.bytes) : -1;
-						double msc = (metrics.bytes > 0) ? ((double)(100 * metrics.scan) / (double)metrics.bytes) : -1;
-						double ekb = (metrics.bytes > 0) ? ((double)(1024 * metrics.errors) / (double)metrics.bytes) : -1;
-						System.out.println(String.format(" : %8.3f mb/s %7.3f nul/kb %6.3f%% m+ %6.3f%% m* %6.3f%% m-", mbps, ekb, mps, mpr, msc));
+						long bytes = metrics.traps[0][1] + metrics.traps[1][1] + metrics.traps[2][1] + metrics.traps[3][1];
+						double mbps = (tjrte > 0) ? (double)(bytes*1000000000l) / (double)(tjrte*1024*1024) : -1;
+						double mnone = (bytes > 0) ? ((double) (100 * metrics.traps[0][1]) / (double) bytes) : -1;
+						double mps = (bytes > 0) ? ((double) (100 * metrics.traps[1][1]) / (double) bytes) : -1;
+						double mpr = (bytes > 0) ? ((double)(100 * metrics.traps[2][1]) / (double)bytes) : -1;
+						double msc = (bytes > 0) ? ((double)(100 * metrics.traps[3][1]) / (double)bytes) : -1;
+						double ekb = (bytes > 0) ? ((double)(1024 * metrics.errors) / (double)bytes) : -1;
+						long none = metrics.traps[0][0] > 0 ? metrics.traps[0][1] / metrics.traps[0][0] : 0;
+						long sum = metrics.traps[1][0] > 0 ? metrics.traps[1][1] / metrics.traps[1][0] : 0;
+						long product = metrics.traps[2][0] > 0 ? metrics.traps[2][1] / metrics.traps[2][0] : 0;
+						long scan = metrics.traps[3][0] > 0 ? metrics.traps[3][1] / metrics.traps[3][0] : 0;
+						String snone= String.format("(%d/%.2f%%):none", none, mnone);
+						String ssum = String.format("(%d/%.2f%%):msum", sum, mps);
+						String sproduct = String.format("(%d/%.2f%%):mproduct", product, mpr);
+						String sscan = String.format("(%d/%.2f%%):mscan", scan, msc);
+						System.out.println(String.format("%8.3f mb/s %7.3f nul/kb %16s %16s %20s %17s", 
+							mbps, ekb, snone, ssum, sproduct, sscan));
 					} else {
 						try (
 							final FileInputStream isr = new FileInputStream(f);
@@ -172,13 +182,12 @@ public static void main(final String[] args) {
 						}
 						t1 = System.nanoTime() - t0;
 						if (i >= 10) {
-							System.out.print(String.format("%4d", count > 0 ? t1/1000000 : -1));
 							tregex += t1;
 						}
 						assert count > 0;
 					}
 					double mbps = (tregex > 0) ? (double)((loops - 10)*blen*1000) / (double)tregex : -1;
-					System.out.println(String.format(" : %8.3f mb/s", mbps));
+					System.out.println(String.format("%8.3f mb/s", mbps));
 				} else {
 					int count = 0;
 					final byte[] delimiters = new byte[] {'|','\n'};

src/com/characterforming/jrte/test/TestRunner.java

@@ -82,7 +82,6 @@ public static void main(final String[] args) {
 						}
 						t1 = System.nanoTime() - t0;
 						if (i >= 10) {
-							System.out.print(String.format("%4d", t1/1000000));
 							t2 += t1;
 						}
 						assert !trex.status().isRunnable();
@@ -91,7 +90,7 @@ public static void main(final String[] args) {
 					}
 				}
 				double mbps = (t2 > 0) ? (double)(100000000l*1000000000l) / (double)(t2*1024*1024) : -1;
-				System.out.println(String.format(" : %8.3f mb/s (bytes)", mbps));
+				System.out.println(String.format("%8.3f mb/s (bytes)", mbps));
 			}
 			exitCode = 0;
 		} catch (Exception e) {

src/com/characterforming/ribose/IParameterizedEffector.java

@@ -52,13 +52,13 @@
  * {@link BaseParameterizedEffector}, which all {@code IParameterizedEffector}
  * implementations must extend:
  * <ol>
- * <li> model calls <i>subclass</i>.allocateParameters(int n)
+ * <li> model calls <i>subclassEffector</i>.allocateParameters(int n)
  * <ul><li>allocates new P[n]</ul>
- * <li> model calls <i>superclass</i>.compileParameters(IToken[][], List)
- * <ul><li>sets P[i] = <i>subclass</i>Effector.compileParameter(IToken[]) for i&lt;n</ul> 
- * <li> model calls <i>subclass</i>Effector.passivate()
- * <ul><li>if overridden, <i>subclass</i> must call <i>superclass</i>.passivate()
- * 	<ul><li>BaseEffector.target = null<li>BaseEffector.output = null</ul>
+ * <li> model calls <i>superclassEffector</i>.compileParameters(IToken[][], List)
+ * <ul><li>sets P[i] = <i>subclassEffector</i>.compileParameter(IToken[]) for i&lt;n</ul> 
+ * <li> model calls <i>subclassEffector</i>.passivate()
+ * <ul><li>if overridden, <i>subclassEffector</i> must call <i>superclassEffector</i>.passivate()
+ * 	<ul><li><i>superclassEffector</i>.target = null<li><i>superclassEffector</i>.output = null</ul>
  * </ul> 
  * </ol>
  * Proxy effectors may receive calls to {@link #showParameterType()} and {@link