Fixes #40, fixes #41

- iToken (interface changes)
  - removed getType(), getReference()
  - added isLiteral(), isField(), isSignal(), isTransducer()
- issue fixes and dependent changes in other files
- added .bin (VScode build output) to .gitignore

Signed-off-by: jrte <jrte.project@gmail.com>

.gitignore

@@ -8,6 +8,7 @@ regression.*
 **/.fuse_hidden*
 **/.~*
 **/*.bak
+.bin
 .data/
 .doc/
 .tmp/

build.xml

@@ -48,9 +48,10 @@
 	<property name="regex.linuxkernel" value="([JFMASOND][a-z]+ [0-9]+ (?:[0-9]+:)+[0-9]+) ([-.:A-Za-z_0-9]*) kernel: \[[ ]*[0-9]+\.[0-9]+\] (DROPPED|ABORTED|LIMITED) IN=([-.:A-Za-z_0-9]*) OUT=([-.:A-Za-z_0-9]*)(?: MAC=([-.:A-Za-z_0-9]*))? SRC=([-.:A-Za-z_0-9]*) DST=([-.:A-Za-z_0-9]*).* PROTO=([-.:A-Za-z_0-9]*)(?:.* SPT=([-.:A-Za-z_0-9]*) DPT=([-.:A-Za-z_0-9]*))?"/>
 	<property name="ribose.inbuffer.size" value="65536"/>
 	<property name="ribose.outbuffer.size" value="65536"/>
-	<property name="ribose.sum.threshold" value="128 (default)"/>
-	<property name="ribose.product.threshold" value="10 (default)"/>
-	<property name="jargs.ribose.compile" value=""/>
+	<property name="ribose.sum.threshold" value="128"/>
+	<property name="ribose.product.threshold" value="10"/>
+	<property name="jargs.ribose.compile" value="-ea -Dribose.product.threshold=${ribose.product.threshold} -Dribose.sum.threshold=${ribose.sum.threshold}"/>
+	<!--property name="jargs.ribose.compile" value="-ea"/-->
 	<property name="jargs.jit.diagnostic" value="-XX:+UnlockDiagnosticVMOptions -XX:+PrintCompilation -XX:+PrintInlining"/>
 	<property name="jargs.jit.inline" value="-XX:CompileCommandFile=etc/jit/ribose.jit"/>
 	<property name="jargs.gc.buffers" value="-Dribose.inbuffer.size=${ribose.inbuffer.size} -Dribose.outbuffer.size=${ribose.outbuffer.size}"/>

patterns/test/ValuesTest.inr

@@ -9,7 +9,7 @@ ValuesTest = (
 		|	(dot, paste) (digit, paste)* (space|nl, real paste out clear)
 		| ((black - {digit,dash,dot}), paste) (black, paste) (space|nl, string paste out clear)
 		)
-		((pass, out[`!!pass; `]) | (fail, out[`!!fail; `]))
+		((pass, out[`\xF8!pass; `]) | (fail, out[`\xF8!fail; `]))
 	)*
 ):dfamin;
 

src/com/characterforming/jrte/engine/Assembler.java

@@ -119,15 +119,18 @@ Assembly assemble(final int[][][] transitionMatrix, HashMap<Ints, Integer> effec
 						} else if (nextState.isSumState()) {
 							transition[1] = this.injectSumEffector(nextState.idempotentBytes,
 								fst.matrix(), state, eq);
-						} else if (nextState.isProductState()
-						&& this.walk(nextState, walkedStates, fst, walkResult).get(0) >= ModelCompiler.MIN_PRODUCT_LENGTH) {
-							assert nextState.idempotentCount >= 255;
-							transition[1] = this.injectProductEffector(this.product(walkResult), walkResult.get(1),
-								fst.matrix(), state, eq);
+						} else if (nextState.isProductState()) {
+							this.walk(nextState, walkedStates, fst, walkResult);
+							if (walkResult.get(0) >= ModelCompiler.MIN_PRODUCT_LENGTH) {
+								assert nextState.idempotentCount >= 255;
+								transition[0] = walkResult.get(1);
+								transition[1] = this.injectProductEffector(walkResult,
+									fst.matrix(),state, eq);
+							}
 						}
+						if (transition[1] < 0)
+							markedEffects[-1 * transition[1]] = true;
 					}
-					if (transition[1] < 0)
-						markedEffects[-1 * transition[1]] = true;
 				}
 			}
 
@@ -221,10 +224,9 @@ private Fst reduceEquivalentInputs(int[][][] transitionMatrix) {
 			assert transitionMatrix[token].length == transitionMatrix[0].length;
 			final IntsArray transitions = new IntsArray(transitionMatrix[token]);
 			HashSet<Integer> equivalentInputOrdinals = equivalenceSets.computeIfAbsent(
-					transitions, absent -> new HashSet<>(10));
-			if (equivalentInputOrdinals.isEmpty()) {
+				transitions, absent -> new HashSet<>(10));
+			if (equivalentInputOrdinals.isEmpty())
 				equivalenceSets.put(transitions, equivalentInputOrdinals);
-			}
 			equivalentInputOrdinals.add(token);
 		}
 
@@ -245,7 +247,7 @@ private Fst reduceEquivalentInputs(int[][][] transitionMatrix) {
 		for (int state = 0; state < nStates; state++) {
 			for (int eq = 0; eq < nInputs; eq++)
 				matrix[state][eq] = transitionMatrix[equiv[eq].iterator().next().intValue()][state];
-			states[state] = new State(state, matrix[state],
+					states[state] = new State(state, matrix[state],
 					equiv, this.compiler.getSignalLimit());
 		}
 
@@ -259,11 +261,12 @@ private int mark(int[][][] matrix, boolean[] markedStates) {
 		int marked = 0;
 		while (!stack.isEmpty()) {
 			int state = stack.pop();
-			markedStates[state] = true;
-			for (int[] transition : matrix[state])
-				if (!markedStates[transition[0]])
+			if (!markedStates[state]) {
+				markedStates[state] = true;
+				for (int[] transition : matrix[state])
 					stack.push(transition[0]);
-			++marked;
+				++marked;
+			}
 		}
 		return marked;
 	}
@@ -278,15 +281,14 @@ private int injectEffector(int action, int effector, int parameter) {
 				Transducer.parameter(action),
 				-1 * effector, parameter, 0 };
 		} else if (action > NUL) {
-			key = new int[] {
-				action, -1 * effector, parameter, 0 };
+			key = new int[] { action, -1 * effector, parameter, 0 };
 		} else if (action < NUL) {
 			key = this.effectVectors.get(-1 * action);
 			key = Arrays.copyOf(key, key.length + 2);
 			key[key.length - 3] = -1 * effector;
 			key[key.length - 2] = parameter;
 			key[key.length - 1] = 0;
-		} else key = null;
+		}
 		if (key != null) {
 			action = -1 * this.effectVectorMap.computeIfAbsent(
 				new Ints(key), absent -> this.effectVectorMap.size());
@@ -296,32 +298,33 @@ private int injectEffector(int action, int effector, int parameter) {
 		return action;
 	}
 
-	private int injectScanEffector(int idempotentByte, int[][][] matrix, State state, int eq) {
-		assert idempotentByte >= 0 && idempotentByte < 256;
-		byte scanByte = (byte)(idempotentByte & 0xff);
-		Argument argument = new Argument(-1,
-			new BytesArray(new byte[][] { { scanByte } }));
+	private int injectScanEffector(int token, int[][][] matrix, State state, int eq) {
+		byte[] scan = new byte[] { Token.escape(), (byte) (token & 0xff) };
+		Argument argument = new Argument(-1, new BytesArray(new byte[][] { scan }));
 		return this.injectEffector(matrix[state.ordinal][eq][1], mscanOrdinal,
 			this.compiler.compileParameters(mscanOrdinal, argument));
 	}
 
-	private int injectSumEffector(long[] idempotentBitmap, int[][][] matrix, State state, int eq) {
-		int selfCount = 0;
-		byte[] selfBytes = new byte[256];
-		for (int word = 0; word < idempotentBitmap.length; word++)
+	private int injectSumEffector(long[] bitmap, int[][][] matrix, State state, int eq) {
+		int n = 0;
+		byte[] sum = new byte[256];
+		sum[n++] = Token.escape();
+		for (int word = 0; word < bitmap.length; word++)
 			for (int bit = 0; bit < 64; bit++)
-				if (0 != ((1L << bit) & idempotentBitmap[word]))
-					selfBytes[selfCount++] = (byte)(64 * word + bit);
-		Argument argument = new Argument(-1,
-			new BytesArray(new byte[][] { Arrays.copyOf(selfBytes, selfCount) }));
+				if (0 != ((1L << bit) & bitmap[word]))
+					sum[n++] = (byte) (64 * word + bit);
+		sum = Arrays.copyOf(sum, n);
+		Argument argument = new Argument(-1, new BytesArray(new byte[][] { sum }));
 		return this.injectEffector(matrix[state.ordinal][eq][1], msumOrdinal,
 			this.compiler.compileParameters(msumOrdinal, argument));
 	}
 
-	private int injectProductEffector(byte[] product, int endpoint, int[][][] matrix, State state, int eq) {
-		Argument argument = new Argument(-1,
-			new BytesArray(new byte[][] { Arrays.copyOf(product, product.length) }));
-		matrix[state.ordinal][eq][0] = endpoint;
+	private int injectProductEffector(ArrayList<Integer> walkResult, int[][][] matrix, State state, int eq) {
+		byte[] product = new byte[walkResult.size() - 2];
+		product[0] = Token.escape();
+		for (int i = 1; i < product.length; i++)
+			product[i] = (byte) (walkResult.get(i + 1).intValue() & 0xff);
+		Argument argument = new Argument(-1, new BytesArray(new byte[][] { product }));
 		return this.injectEffector(matrix[state.ordinal][eq][1], mproductOrdinal,
 			this.compiler.compileParameters(mproductOrdinal, argument));
 	}
@@ -331,8 +334,19 @@ private ArrayList<Integer> walk(State nextState, boolean[] walkedStates, Fst fst
 		walkResult.add(0); walkResult.add(-1);
 		ArrayList<Integer> walkStates = new ArrayList<>(16);
 		Arrays.fill(walkedStates, false);
+		int[] transition = nextState.transitions[fst.inputEquivalenceIndex[Signal.NUL.signal()]];
+		int[] tx = new int[] { transition[0] != nextState.ordinal ? transition[0] : -1, transition[1] };
+		int[] ty = new int[] { Integer.MIN_VALUE, Integer.MIN_VALUE };
 		while (nextState.isProductState() && !walkedStates[nextState.ordinal]) {
 			assert nextState.outboundByte == (nextState.outboundByte & 0xff);
+			transition = nextState.transitions[fst.inputEquivalenceIndex[Signal.NUL.signal()]];
+			ty[0] = transition[0] != nextState.ordinal ? transition[0] : -1;
+			ty[1] = transition[1];
+			if ((tx[0] != ty[0]) || (tx[1] != ty[1])) {
+				while (walkResult.size() > 2)
+					walkResult.remove(2);
+				return walkResult;
+			}
 			walkResult.add(nextState.outboundByte);
 			walkedStates[nextState.ordinal] = true;
 			walkStates.add(nextState.ordinal);
@@ -346,13 +360,6 @@ private ArrayList<Integer> walk(State nextState, boolean[] walkedStates, Fst fst
 		return walkResult;
 	}
 
-	private byte[] product(ArrayList<Integer> walkResult) {
-		byte[] p = new byte[walkResult.size() - 3];
-		for (int i = 0; i < p.length; i++)
-			p[i] = (byte)(walkResult.get(i + 2).intValue() & 0xff);
-		return p;
-	}
-
 	@SuppressWarnings("unchecked")
 	private HashSet<Integer>[] allocateHashSetArray(int size) {
 		return (HashSet<Integer>[])new HashSet<?>[size];

src/com/characterforming/jrte/engine/BaseFieldEffector.java

@@ -1,18 +1,18 @@
 /***
  * Ribose is a recursive transduction engine for Java
- * 
+ *
  * Copyright (C) 2011,2022 Kim Briggs
- * 
+ *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
- * 
+ *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU General Public License
  * along with this program (LICENSE-gpl-3.0). If not, see
  * <http://www.gnu.org/licenses/#GPL>.
@@ -30,14 +30,14 @@
  * Base class for parameterized field effectors, which are invoked with
  * field name parameters. The setParamater(int, charset, byte[][]), invoke(), and
  * invoke(int) methods must be implemented by subclasses.
- * 
+ *
  * @author Kim Briggs
  */
 abstract class BaseFieldEffector extends BaseParameterizedEffector<Transductor, Integer> {
 	/**
 	 * Constructor
-	 * 
-	 * @param transductor The transductor target that binds the effector  
+	 *
+	 * @param transductor The transductor target that binds the effector
 	 * @param name the field name
 	 * @throws CharacterCodingException
 	 */
@@ -53,10 +53,10 @@ public Integer[] allocateParameters(int parameterCount) {
 	@Override // IParameterizedEffector#compileParameter(IToken[])
 	public Integer compileParameter(final IToken[] parameterList) throws TargetBindingException {
 		if (parameterList.length != 1) {
-			throw new TargetBindingException(String.format("%1$s.%2$s[]: effector accepts exactly one parameter", 
+			throw new TargetBindingException(String.format("%1$s.%2$s[]: effector accepts exactly one parameter",
 				super.target.getName(), super.getName()));
 		}
-		if (parameterList[0].getType() != IToken.Type.FIELD) {
+		if (!parameterList[0].isField()) {
 			throw new TargetBindingException(String.format("%1$s.%2$s[]: effector accepts only a FIELD parameter",
 				super.target.getName(), super.getName()));
 		}

src/com/characterforming/jrte/engine/BaseInputOutputEffector.java

@@ -60,11 +60,11 @@ public IToken[] compileParameter(final IToken[] parameterList) throws TargetBind
 				super.target.getName(), super.getName()));
 		}
 		for (IToken token : parameterList) {
-			if (token.getType() != IToken.Type.LITERAL && token.getType() != IToken.Type.FIELD) {
-					throw new TargetBindingException(String.format(
-						"%1$s.%2$s[]: literal or field name expected, found '%3$s'",
-						super.target.getName(), super.getName().asString(),
-						token.asString()));
+			if (!token.isLiteral() && !token.isField()) {
+				throw new TargetBindingException(String.format(
+					"%1$s.%2$s[]: literal or field name expected, found '%3$s'",
+					super.target.getName(), super.getName().asString(),
+					token.asString()));
 			}
 		}
 		return parameterList;

src/com/characterforming/jrte/engine/ModelCompiler.java

@@ -45,7 +45,6 @@
 import com.characterforming.ribose.ITarget;
 import com.characterforming.ribose.ITransduction;
 import com.characterforming.ribose.ITransductor;
-import com.characterforming.ribose.IToken.Type;
 import com.characterforming.ribose.base.BaseEffector;
 import com.characterforming.ribose.base.Bytes;
 import com.characterforming.ribose.base.Codec;
@@ -62,8 +61,8 @@ public final class ModelCompiler extends Model implements ITarget, AutoCloseable
 	private static final String AUTOMATON = "Automaton";
 	private static final String AMBIGUOUS_STATE_MESSAGE = "%1$s: Ambiguous state %2$d";
 
-	static final int MIN_PRODUCT_LENGTH = Integer.parseInt(System.getProperty("ribose.product.threshold", "10"));
-	static final int MIN_SUM_SIZE = Integer.parseInt(System.getProperty("ribose.sum.threshold", "128"));
+	static final int MIN_PRODUCT_LENGTH = Integer.parseInt(System.getProperty("ribose.product.threshold", "-1"));
+	static final int MIN_SUM_SIZE = Integer.parseInt(System.getProperty("ribose.sum.threshold", "-1"));
 	static final int MIN_SCAN_SIZE = 255;
 
 	private Bytes transducerName;
@@ -405,7 +404,7 @@ private boolean compileTransducer(File inrFile) {
 
 	private boolean validate() {
 		for (Token token : this.tapeTokens.get(0)) {
-			if (token.getType() != Type.LITERAL && token.getType() != Type.SIGNAL) {
+			if (!token.isLiteral() && !token.isSignal()) {
 				this.addError(String.format("Error: Invalid %2$s token '%1$s' on tape 0",
 					token.asString(), token.getTypeName()));
 			} else if (token.getSymbol().bytes().length > 1
@@ -415,7 +414,7 @@ private boolean validate() {
 			}
 		}
 		for (Token token : this.tapeTokens.get(1)) {
-			if (token.getType() != Type.LITERAL) {
+			if (!token.isLiteral()) {
 				this.addError(String.format("Error: Invalid %2$s token '%1$s' on tape 1",
 					token.asString(), token.getTypeName()));
 			} else if (this.getEffectorOrdinal(token.getSymbol()) < 0) {
@@ -424,11 +423,11 @@ private boolean validate() {
 			}
 		}
 		for (Token token : this.tapeTokens.get(2)) {
-			if (token.getType() == Type.TRANSDUCER
+			if (token.isTransducer()
 				&& super.getTransducerOrdinal(token.getSymbol()) < 0) {
 				this.addError(String.format("Error: Unrecognized transducer token '%1$s' on tape 1",
 					token.asString()));
-			} else if (token.getType() == Type.SIGNAL
+			} else if (token.isSignal()
 				&& super.getSignalOrdinal(token.getSymbol()) > Signal.EOS.signal()
 				&& !this.tapeTokens.get(0).contains(token)) {
 				this.addError(String.format("Error: Signal token '%1$s' on tape 2 is never referenced on tape 0",
@@ -510,18 +509,17 @@ void putTransition(Transition transition) {
 				if (transition.tape == 1 || transition.symbol.getLength() > 1) {
 					Token token = new Token(transition.symbol.bytes(), -1, transducerOrdinal);
 					Bytes symbol = token.getSymbol();
-					Type type = token.getType();
-					if (transition.tape == 0 && type == Type.LITERAL && transition.symbol.getLength() > 1) {
-						super.addSignal(symbol);
-						this.tapeTokens.get(0).add(new Token(token.getReference(Type.SIGNAL, symbol.bytes())));
+					if (transition.tape == 0 && token.isLiteral() && transition.symbol.getLength() > 1) {
+						this.tapeTokens.get(0).add(new Token(Token.reference(Token.Type.SIGNAL, symbol.bytes()),
+							super.addSignal(symbol), transducerOrdinal));
 					} else if (transition.tape == 1) {
 						this.tapeTokens.get(1).add(token);
-					} else if (transition.tape == 2 && type != Type.LITERAL) {
-						if (type == Type.FIELD) {
+					} else if (transition.tape == 2 && !token.isLiteral()) {
+						if (token.isField()) {
 							token.setOrdinal(super.addLocalField(this.transducerOrdinal, super.addField(symbol)));
-						} else if (type == Type.TRANSDUCER) {
+						} else if (token.isTransducer()) {
 							token.setOrdinal(super.addTransducer(symbol));
-						} else if (type == Type.SIGNAL) {
+						} else if (token.isSignal()) {
 							token.setOrdinal(super.addSignal(symbol));
 						}
 						this.tapeTokens.get(2).add(token);