From 87f6054804a9e27dfa3e0c8ddeaf1382f7307d56 Mon Sep 17 00:00:00 2001 From: jrte Date: Sun, 8 Oct 2023 11:03:10 -0300 Subject: [PATCH] Store codecs in ThreadLocal - added Codec class to maintain codecs in thread local variable - all threads attach Codec instance on first Codec use - thread local variable detached from thread calling IModel.close() - or when model autoclosed after try-with-resources involving thread - or when thread explicitly calls static IModel.detach() Signed-off-by: jrte --- build.xml | 2 +- .../jrte/engine/Automaton.java | 5 +- .../characterforming/jrte/engine/Base.java | 27 --- .../jrte/engine/BaseFieldEffector.java | 5 +- .../jrte/engine/BaseInputOutputEffector.java | 16 +- .../characterforming/jrte/engine/Codec.java | 105 +++++++++++ .../characterforming/jrte/engine/Model.java | 68 +++---- .../jrte/engine/ModelCompiler.java | 94 ++++++---- .../jrte/engine/ModelLoader.java | 22 ++- .../characterforming/jrte/engine/Token.java | 9 +- .../jrte/engine/Transductor.java | 166 ++++++++---------- .../jrte/test/FileRunner.java | 40 +---- .../jrte/test/TestRunner.java | 5 +- .../jrte/test/TestTarget.java | 39 ++-- .../characterforming/ribose/IEffector.java | 14 +- src/com/characterforming/ribose/IModel.java | 20 ++- src/com/characterforming/ribose/IOutput.java | 50 +++--- .../ribose/IParameterizedEffector.java | 15 +- src/com/characterforming/ribose/ITarget.java | 3 +- src/com/characterforming/ribose/IToken.java | 11 +- .../characterforming/ribose/ITransductor.java | 9 +- src/com/characterforming/ribose/Ribose.java | 16 +- .../ribose/base/BaseEffector.java | 35 +--- .../base/BaseParameterizedEffector.java | 12 +- .../characterforming/ribose/base/Bytes.java | 134 +++++--------- .../ribose/base/SimpleTarget.java | 3 +- 26 files changed, 473 insertions(+), 452 deletions(-) create mode 100644 src/com/characterforming/jrte/engine/Codec.java diff --git a/build.xml b/build.xml index 6dc4e7a..79a23b0 100755 --- a/build.xml +++ b/build.xml @@ -203,7 +203,7 @@ diff --git a/src/com/characterforming/jrte/engine/Automaton.java b/src/com/characterforming/jrte/engine/Automaton.java index 6139470..c2dda5c 100644 --- a/src/com/characterforming/jrte/engine/Automaton.java +++ b/src/com/characterforming/jrte/engine/Automaton.java @@ -26,6 +26,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.charset.CharacterCodingException; import java.text.ParseException; import java.util.Arrays; import java.util.logging.Level; @@ -79,7 +80,7 @@ boolean assemble(File inrAutomataDirectory) { if (commit) { this.compiler.saveTransducer(); } - } catch (ModelException | ParseException e) { + } catch (ModelException | ParseException | CharacterCodingException e) { String msg = String.format("%1$s: Exception caught assembling compiler model file; %2$s", filename, e.getMessage()); rtcLogger.log(Level.SEVERE, msg, e); @@ -91,7 +92,7 @@ boolean assemble(File inrAutomataDirectory) { return commit; } - private boolean parse(byte[] dfa) throws ParseException { + private boolean parse(byte[] dfa) throws ParseException, CharacterCodingException { this.dfain.clear(dfa); this.line = 0; diff --git a/src/com/characterforming/jrte/engine/Base.java b/src/com/characterforming/jrte/engine/Base.java index 85bc6e0..5a8c929 100644 --- a/src/com/characterforming/jrte/engine/Base.java +++ b/src/com/characterforming/jrte/engine/Base.java @@ -20,10 +20,6 @@ package com.characterforming.jrte.engine; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CodingErrorAction; import java.util.logging.FileHandler; import java.util.logging.Handler; import java.util.logging.Level; @@ -57,7 +53,6 @@ private Base() { private static final int FILE_LOGGER_LIMIT = 1024 * 1024; private static final int INPUT_BUFFER_SIZE = Integer.parseInt(System.getProperty("ribose.inbuffer.size", "65536")); private static final int OUTPUT_BUFFER_SIZE = Integer.parseInt(System.getProperty("ribose.outbuffer.size", "8196")); - private static final Charset runtimeCharset = Charset.forName(System.getProperty("ribose.runtime.charset", "UTF-8")); private static final Logger rtcLogger = Logger.getLogger("ribose-compile"); private static final Logger rteLogger = Logger.getLogger("ribose-runtime"); @@ -98,28 +93,6 @@ public static void endLogging() { Base.endLogger(Base.rteLogger); } - /** - * Instantiate a new {@code CharsetDecoder}. All textual data in ribose models - * are represented in encoded form (eg, UTF-8 byte arrays). - * - * @return a new CharsetDecoder insstance - */ - public static CharsetDecoder newCharsetDecoder() { - return Base.runtimeCharset.newDecoder() - .onUnmappableCharacter(CodingErrorAction.REPLACE) - .onMalformedInput(CodingErrorAction.REPLACE); - } - - /** - * Instantiate a new {@code CharsetEncoder}. All textual data in ribose models - * are represented in encoded form (eg, UTF-8 byte arrays). - * - * @return a new CharsetEncoder instance - */ - public static CharsetEncoder newCharsetEncoder() { - return Base.runtimeCharset.newEncoder(); - } - /** * Get the size (in bytes) to use for input buffers. * diff --git a/src/com/characterforming/jrte/engine/BaseFieldEffector.java b/src/com/characterforming/jrte/engine/BaseFieldEffector.java index 3a64764..19aa4c2 100644 --- a/src/com/characterforming/jrte/engine/BaseFieldEffector.java +++ b/src/com/characterforming/jrte/engine/BaseFieldEffector.java @@ -20,6 +20,8 @@ package com.characterforming.jrte.engine; +import java.nio.charset.CharacterCodingException; + import com.characterforming.ribose.IToken; import com.characterforming.ribose.base.BaseParameterizedEffector; import com.characterforming.ribose.base.TargetBindingException; @@ -37,8 +39,9 @@ abstract class BaseFieldEffector extends BaseParameterizedEffector. + */ + +package com.characterforming.jrte.engine; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; + +import com.characterforming.ribose.base.Bytes; + +public class Codec { + private static final Charset CHARSET = Charset.forName(System.getProperty("ribose.runtime.charset", "UTF-8")); + private static final ThreadLocal LOCAL = ThreadLocal.withInitial(Codec::new); + private CharsetDecoder decoder; + private CharsetEncoder encoder; + + private Codec() { + this.decoder = Codec.CHARSET.newDecoder() + .onUnmappableCharacter(CodingErrorAction.REPLACE) + .onMalformedInput(CodingErrorAction.REPLACE) + .replaceWith("~") + .reset(); + this.encoder = Codec.CHARSET.newEncoder() + .onUnmappableCharacter(CodingErrorAction.REPLACE) + .onMalformedInput(CodingErrorAction.REPLACE) + .replaceWith("~".getBytes()) + .reset(); + } + + private static Codec set() { + Codec codec = new Codec(); + Codec.LOCAL.set(codec); + return codec; + } + + private static Codec get() { + Codec codec = Codec.LOCAL.get() ; + return (codec == null) ? Codec.set() : codec; + } + + public static void detach() { + Codec.LOCAL.remove(); + } + + /** + * Decode UTF-8 bytes to a String. + * + * @param bytes the bytes to decode + * @param length the number of bytes to decode + * @return a String containing the decoded text + * @throws CharacterCodingException if decoding fails + */ + public static String decode(final byte[] bytes, final int length) throws CharacterCodingException { + assert 0 <= length && length <= bytes.length; + int size = Math.max(Math.min(length, bytes.length), 0); + return Codec.get().decoder.reset().decode(ByteBuffer.wrap(bytes, 0, size)).toString(); + } + + /** + * Decode UTF-8 bytes to a String. + * + * @param bytes the bytes to decode + * @return a String containing the decoded text + * @throws CharacterCodingException if decoding fails + */ + public static String decode(final byte[] bytes) throws CharacterCodingException { + return Codec.decode(bytes, bytes.length); + } + + /** + * Encode a String. + * + * @param chars the string to encode + * @return the encoded Bytes + * @throws CharacterCodingException if encoding fails + */ + public static Bytes encode(final String chars) throws CharacterCodingException { + ByteBuffer buffer = Codec.get().encoder.reset().encode(CharBuffer.wrap(chars.toCharArray())); + byte[] bytes = new byte[buffer.limit()]; + buffer.get(bytes, 0, bytes.length); + return new Bytes(bytes); + } +} diff --git a/src/com/characterforming/jrte/engine/Model.java b/src/com/characterforming/jrte/engine/Model.java index 6857aa2..db4ad2a 100644 --- a/src/com/characterforming/jrte/engine/Model.java +++ b/src/com/characterforming/jrte/engine/Model.java @@ -28,8 +28,7 @@ import java.io.RandomAccessFile; import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; +import java.nio.charset.CharacterCodingException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -84,8 +83,6 @@ public record Argument(int transducerOrdinal, BytesArray tokens) {} protected final Class targetClass; protected final Logger rtcLogger; protected final Logger rteLogger; - protected final CharsetDecoder decoder; - protected final CharsetEncoder encoder; protected HashMap signalOrdinalMap; protected HashMap fieldOrdinalMap; protected HashMap effectorOrdinalMap; @@ -106,8 +103,6 @@ public Model() { this.targetMode = TargetMode.PROXY_COMPILER; this.rtcLogger = Base.getCompileLogger(); this.rteLogger = Base.getRuntimeLogger(); - this.decoder = Base.newCharsetDecoder(); - this.encoder = Base.newCharsetEncoder(); this.modelVersion = Base.RTE_VERSION; this.targetClass = this.getClass(); this.deleteOnClose = false; @@ -119,8 +114,6 @@ protected Model(final File modelPath, Class targetClass, TargetMode targetMod this.targetMode = targetMode; this.rtcLogger = Base.getCompileLogger(); this.rteLogger = Base.getRuntimeLogger(); - this.decoder = Base.newCharsetDecoder(); - this.encoder = Base.newCharsetEncoder(); this.modelVersion = Base.RTE_VERSION; this.modelPath = modelPath; this.deleteOnClose = true; @@ -144,7 +137,7 @@ protected Model(final File modelPath, Class targetClass, TargetMode targetMod this.writeLong(0); this.writeString(this.modelVersion); this.writeString(this.targetClass.getName()); - } catch (FileNotFoundException e) { + } catch (FileNotFoundException | CharacterCodingException e) { throw new ModelException(String.format("Unable to create model file '%s'.", this.modelPath.toPath().toString()), e); } @@ -156,8 +149,6 @@ protected Model(final File modelPath) throws ModelException { this.modelPath = modelPath; this.rtcLogger = Base.getCompileLogger(); this.rteLogger = Base.getRuntimeLogger(); - this.decoder = Base.newCharsetDecoder(); - this.encoder = Base.newCharsetEncoder(); this.transducerFieldMaps = new HashMap<>(256); this.deleteOnClose = false; String targetClassname = "?"; @@ -215,6 +206,7 @@ protected void close() { this.modelPath.getPath())); this.deleteOnClose |= this.targetMode.isLive(); } finally { + Codec.detach(); assert !this.deleteOnClose || this.targetMode.isLive(); if (this.deleteOnClose && this.targetMode.isLive() && this.modelPath.exists() && !this.modelPath.delete()) { @@ -300,7 +292,7 @@ protected boolean save(Argument[][] effectorParameters) { return !this.deleteOnClose; } - protected Model load() throws ModelException { + protected Model load() throws ModelException, CharacterCodingException { this.seek(0); long indexPosition = this.readLong(); final String loadedVersion = this.readString(); @@ -389,7 +381,7 @@ public boolean map(PrintStream mapWriter) { } for (int i = 0; i < signalIndex.length; i++) { mapWriter.printf("%1$6d signal %2$s%n", i + Base.RTE_SIGNAL_BASE, - signalIndex[i].toString(this.getDecoder())); + signalIndex[i].asString()); } Bytes[] fieldIndex = new Bytes[this.fieldOrdinalMap.size()]; for (Map.Entry m : this.fieldOrdinalMap.entrySet()) { @@ -401,7 +393,7 @@ public boolean map(PrintStream mapWriter) { } for (int transducerOrdinal = 0; transducerOrdinal < transducerIndex.length; transducerOrdinal++) { mapWriter.printf("%1$6d transducer %2$s%n", transducerOrdinal, - transducerIndex[transducerOrdinal].toString(this.getDecoder())); + transducerIndex[transducerOrdinal].asString()); Map fieldMap = this.transducerFieldMaps.get(transducerOrdinal); Bytes[] fields = new Bytes[fieldMap.size()]; for (Entry e : fieldMap.entrySet()) { @@ -409,7 +401,7 @@ public boolean map(PrintStream mapWriter) { } for (int field = 0; field < fields.length; field++) { mapWriter.printf("%1$6d field ~%2$s%n", field, - fields[field].toString(this.getDecoder())); + fields[field].asString()); } } Bytes[] effectorIndex = new Bytes[this.effectorOrdinalMap.size()]; @@ -418,11 +410,11 @@ public boolean map(PrintStream mapWriter) { } for (int effector = 0; effector < effectorIndex.length; effector++) { mapWriter.printf("%1$6d effector %2$s", effector, - effectorIndex[effector].toString(this.getDecoder())); + effectorIndex[effector].asString()); if (this.proxyEffectors[effector] instanceof BaseParameterizedEffector proxyEffector) { mapWriter.printf(" [ %1$s ]%n", proxyEffector.showParameterType()); for (int parameter = 0; parameter < proxyEffector.getParameterCount(); parameter++) { - mapWriter.printf("%1$6d parameter %2$s%n", parameter, proxyEffector.showParameterTokens(this.getDecoder(), parameter)); + mapWriter.printf("%1$6d parameter %2$s%n", parameter, proxyEffector.showParameterTokens(parameter)); } } else { mapWriter.println(); @@ -444,14 +436,6 @@ public String getTargetClassname() { return this.targetClass.getName(); } - protected CharsetDecoder getDecoder() { - return this.decoder.reset(); - } - - protected CharsetEncoder getEncoder() { - return this.encoder.reset(); - } - protected boolean checkTargetEffectors(ITarget target, IEffector[] boundFx) { boolean checked = true; if (boundFx.length != this.proxyEffectors.length) { @@ -509,7 +493,7 @@ protected Argument[][] compileModelParameters(List errors) throws Effect for (final Map.Entry entry : effectorMap.entrySet()) { if (this.proxyEffectors[entry.getValue()] == null) { this.rtcLogger.log(Level.SEVERE, () -> String.format("%1$s.%2$s: effector ordinal not found", - this.targetName, entry.getKey().toString(this.getDecoder()))); + this.targetName, entry.getKey().asString())); } } return effectorArguments; @@ -537,14 +521,14 @@ protected File getModelPath() { return this.modelPath; } - protected Integer getInputOrdinal(final byte[] input) throws CompilationException { + protected Integer getInputOrdinal(final byte[] input) throws CompilationException, CharacterCodingException { if (input.length == 1) { return Byte.toUnsignedInt(input[0]); } else { Integer ordinal = this.getSignalOrdinal(new Bytes(input)); if (ordinal < 0) { throw new CompilationException(String.format("Invalid input token %s", - Bytes.decode(this.getDecoder(), input, input.length))); + Codec.decode(input, input.length))); } return ordinal; } @@ -794,16 +778,16 @@ protected Argument[] readArguments() throws ModelException { return (arguments != null) ? arguments : new Argument[] {}; } - protected String readString() throws ModelException { + protected String readString() throws ModelException, CharacterCodingException { byte[] bytes = this.readBytes(); - return Bytes.decode(this.getDecoder(), bytes, bytes.length).toString(); + return Codec.decode(bytes); } - protected String[] readStringArray() throws ModelException { + protected String[] readStringArray() throws ModelException, CharacterCodingException { final byte[][] bytesArray = this.readBytesArray(); final String[] stringArray = new String[bytesArray.length]; for (int i = 0; i < bytesArray.length; i++) { - stringArray[i] = Bytes.decode(this.getDecoder(), bytesArray[i], bytesArray[i].length).toString(); + stringArray[i] = Codec.decode(bytesArray[i]); } return stringArray; } @@ -871,22 +855,26 @@ protected long getSafeFilePosition() { } } - protected void writeBytes(final byte[] bytes) throws ModelException { + protected void writeBytes(final byte[] bytes, int length) throws ModelException { final long position = this.getSafeFilePosition(); try { if (bytes != null) { - this.io.writeInt(bytes.length); - this.io.write(bytes); + this.io.writeInt(length); + this.io.write(bytes, 0, length); } else { this.io.writeInt(-1); } } catch (final IOException e) { throw new ModelException(String.format( "Model.writeBytes() IOException at file position %2$d trying to write %1$d bytes starting at file position %3$d", - bytes != null ? bytes.length : 0, this.getSafeFilePosition(), position), e); + bytes != null ? bytes.length : 0, this.getSafeFilePosition(), position), e); } } + protected void writeBytes(final byte[] bytes) throws ModelException { + this.writeBytes(bytes, bytes.length); + } + protected void writeBytes(final ByteBuffer byteBuffer) throws ModelException { if (byteBuffer != null) { byte[] bytes = new byte[byteBuffer.limit() - byteBuffer.position()]; @@ -1009,8 +997,8 @@ protected void writeIntArray(final int[] ints) throws ModelException { } } - protected void writeString(final String s) throws ModelException { - this.writeBytes(Bytes.encode(this.getEncoder(), s).bytes()); + protected void writeString(final String s) throws ModelException, CharacterCodingException { + this.writeBytes(Codec.encode(s).bytes()); } protected void writeTransitionMatrix(final int[][][] matrix) throws ModelException { @@ -1044,9 +1032,9 @@ protected void writeTransitionMatrix(final int[][][] matrix) throws ModelExcepti } protected void writeTransducer(Bytes transducerName, int transducerOrdinal, int[] fields, int[] inputEquivalenceIndex, int[][][] kernelMatrix, int[] effectorVectors) - throws ModelException { + throws ModelException, CharacterCodingException { this.setTransducerOffset(transducerOrdinal, this.seek(-1)); - this.writeString(transducerName.toString(this.getDecoder())); + this.writeString(transducerName.asString()); this.writeInt(transducerOrdinal); this.writeIntArray(fields); this.writeIntArray(inputEquivalenceIndex); diff --git a/src/com/characterforming/jrte/engine/ModelCompiler.java b/src/com/characterforming/jrte/engine/ModelCompiler.java index 30a77bc..f84f0d7 100755 --- a/src/com/characterforming/jrte/engine/ModelCompiler.java +++ b/src/com/characterforming/jrte/engine/ModelCompiler.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.CharacterCodingException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -88,7 +89,7 @@ final class HeaderEffector extends BaseEffector { }; private final int[] fieldOrdinals; - HeaderEffector(ModelCompiler compiler) { + HeaderEffector(ModelCompiler compiler) throws CharacterCodingException { super(compiler, "header"); this.fieldOrdinals = new int[fields.length]; } @@ -96,8 +97,12 @@ final class HeaderEffector extends BaseEffector { @Override public void setOutput(IOutput output) throws EffectorException { super.setOutput(output); - for (int i = 0; i < fields.length; i++) { - this.fieldOrdinals[i] = super.output.getLocalizedFieldIndex(ModelCompiler.AUTOMATON, fields[i]); + try { + for (int i = 0; i < fields.length; i++) { + this.fieldOrdinals[i] = super.output.getLocalizedFieldIndex(ModelCompiler.AUTOMATON, fields[i]); + } + } catch (CharacterCodingException e) { + throw new EffectorException(e); } } @@ -119,7 +124,7 @@ final class TransitionEffector extends BaseEffector { }; private final int[] fieldOrdinals; - TransitionEffector(ModelCompiler compiler) { + TransitionEffector(ModelCompiler compiler) throws CharacterCodingException { super(compiler, "transition"); this.fieldOrdinals = new int[fields.length]; } @@ -127,8 +132,12 @@ final class TransitionEffector extends BaseEffector { @Override public void setOutput(IOutput output) throws EffectorException { super.setOutput(output); - for (int i = 0; i < fields.length; i++) { - this.fieldOrdinals[i] = super.output.getLocalizedFieldIndex(ModelCompiler.AUTOMATON, fields[i]); + try { + for (int i = 0; i < fields.length; i++) { + this.fieldOrdinals[i] = super.output.getLocalizedFieldIndex(ModelCompiler.AUTOMATON, fields[i]); + } + } catch (CharacterCodingException e) { + throw new EffectorException(e); } } @@ -146,13 +155,17 @@ public int invoke() throws EffectorException { } final class AutomatonEffector extends BaseEffector { - AutomatonEffector(ModelCompiler compiler) { + AutomatonEffector(ModelCompiler compiler) throws CharacterCodingException { super(compiler, "automaton"); } @Override public int invoke() throws EffectorException { - ModelCompiler.this.putAutomaton(); + try { + ModelCompiler.this.putAutomaton(); + } catch (CharacterCodingException e) { + throw new EffectorException(e); + } return IEffector.RTX_NONE; } } @@ -198,11 +211,15 @@ public String getName() { @Override // com.characterforming.ribose.IModel.getEffectors() public IEffector[] getEffectors() throws TargetBindingException { - return new IEffector[] { - new HeaderEffector(this), - new TransitionEffector(this), - new AutomatonEffector(this) - }; + try { + return new IEffector[] { + new HeaderEffector(this), + new TransitionEffector(this), + new AutomatonEffector(this) + }; + } catch (CharacterCodingException e) { + throw new TargetBindingException(e); + } } @Override // AutoCloseable.close() @@ -315,10 +332,10 @@ private File lookupCompilerModel() { return compilerModelFile; } - ModelCompiler reset(File inrFile) { + ModelCompiler reset(File inrFile) throws CharacterCodingException { String name = inrFile.getName(); name = name.substring(0, name.length() - Base.AUTOMATON_FILE_SUFFIX.length()); - this.transducerName = Bytes.encode(super.getEncoder(), name); + this.transducerName = Codec.encode(name); this.transducerOrdinal = super.addTransducer(this.transducerName); this.inputStateMap = new HashMap<>(256); this.stateTransitionMap = new HashMap<>(1024); @@ -347,10 +364,11 @@ private boolean createModelFile(File riboseModelFile, Logger rtcLogger) { } private boolean compileTransducer(File inrFile) throws RiboseException { - this.reset(inrFile); int size = (int)inrFile.length(); byte[] bytes = null; + boolean fail = true; try (DataInputStream f = new DataInputStream(new FileInputStream(inrFile))) { + this.reset(inrFile); int position = 0, length = size; bytes = new byte[length]; while (length > 0) { @@ -362,15 +380,14 @@ private boolean compileTransducer(File inrFile) throws RiboseException { } catch (FileNotFoundException e) { this.addError(String.format("%1$s: File not found '%2$s'", this.transducerName, inrFile.getPath())); - return false; + return !fail; } catch (IOException e) { this.addError(String.format("%1$s: IOException compiling '%2$s'; %3$s", this.transducerName, inrFile.getPath(), e.getMessage())); - return false; + return !fail; } - boolean fail = true; try { - Bytes automaton = Bytes.encode(super.getEncoder(), "Automaton"); + Bytes automaton = Codec.encode("Automaton"); if (this.transductor.stop().push(bytes, size).signal(Signal.NIL).start(automaton).status().isRunnable() && this.transductor.run().status().isPaused()) { this.transductor.signal(Signal.EOS).run(); @@ -383,6 +400,9 @@ private boolean compileTransducer(File inrFile) throws RiboseException { } catch (ModelException | EffectorException | DomainErrorException e) { this.addError(String.format("%1$s: Failed to compile '%2$s'; %3$s", this.transducerName, inrFile.getPath(), e.getMessage())); + } catch (CharacterCodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } return !fail; } @@ -391,32 +411,32 @@ private boolean validate() { for (Token token : this.tapeTokens.get(0)) { if (token.getType() != Type.LITERAL && token.getType() != Type.SIGNAL) { this.addError(String.format("Error: Invalid %2$s token '%1$s' on tape 0", - token.getName(super.getDecoder()), token.getTypeName())); + token.asString(), token.getTypeName())); } else if (token.getSymbol().bytes().length > 1 && !this.tapeTokens.get(2).contains(token)) { this.addError(String.format("Error: Unrecognized signal reference '%1$s' on tape 0", - token.getName(super.getDecoder()))); + token.asString())); } } for (Token token : this.tapeTokens.get(1)) { if (token.getType() != Type.LITERAL) { this.addError(String.format("Error: Invalid %2$s token '%1$s' on tape 1", - token.getName(super.getDecoder()), token.getTypeName())); + token.asString(), token.getTypeName())); } else if (this.getEffectorOrdinal(token.getSymbol()) < 0) { this.addError(String.format("Error: Unrecognized effector token '%1$s' on tape 1", - token.getName(super.getDecoder()))); + token.asString())); } } for (Token token : this.tapeTokens.get(2)) { if (token.getType() == Type.TRANSDUCER && super.getTransducerOrdinal(token.getSymbol()) < 0) { this.addError(String.format("Error: Unrecognized transducer token '%1$s' on tape 1", - token.getName(super.getDecoder()))); + token.asString())); } else if (token.getType() == Type.SIGNAL && super.getSignalOrdinal(token.getSymbol()) > Signal.EOS.signal() && !this.tapeTokens.get(0).contains(token)) { this.addError(String.format("Error: Signal token '%1$s' on tape 2 is never referenced on tape 0", - token.getName(super.getDecoder()))); + token.asString())); } } for (Entry e : super.transducerOrdinalMap.entrySet()) { @@ -425,7 +445,7 @@ private boolean validate() { || !super.transducerNameIndex[ordinal].equals(e.getKey()) || super.transducerOffsetIndex[ordinal] <= 0) { this.addError(String.format("'%1$s': referenced but not compiled in model", - e.getKey().toString(this.getDecoder()))); + e.getKey().asString())); } } @@ -436,7 +456,7 @@ private boolean validate() { return !this.hasErrors(); } - void saveTransducer() throws ModelException { + void saveTransducer() throws ModelException, CharacterCodingException { HashMap localFieldMap = this.transducerFieldMaps.get(this.transducerOrdinal); int[] fields = new int[localFieldMap.size()]; for (Entry e : localFieldMap.entrySet()) { @@ -464,7 +484,7 @@ void saveTransducer() throws ModelException { } private String getTransducerName() { - return this.transducerName.toString(super.getDecoder()); + return this.transducerName.asString(); } void putHeader(Header header) { @@ -525,7 +545,7 @@ void putTransition(Transition transition) { } } - void putAutomaton() { + void putAutomaton() throws CharacterCodingException { final Integer[] inrInputStates = this.getInrStates(); if (inrInputStates == null) { this.addError("Empty automaton " + this.getTransducerName()); @@ -602,7 +622,7 @@ private List getErrors() { return this.errors; } - private void factor(final int[][][] transitionMatrix) { + private void factor(final int[][][] transitionMatrix) throws CharacterCodingException { // factor matrix modulo input equivalence final HashMap> rowEquivalenceMap = new HashMap<>((5 * transitionMatrix.length) >> 2); @@ -632,9 +652,9 @@ private void factor(final int[][][] transitionMatrix) { final int nStates = transitionMatrix[0].length; final int nulSignal = Signal.NUL.signal(); final int nulEquivalent = this.inputEquivalenceIndex[nulSignal]; - final int msumOrdinal = super.getEffectorOrdinal(Bytes.encode(super.getEncoder(), "msum")); - final int mproductOrdinal = super.getEffectorOrdinal(Bytes.encode(super.getEncoder(), "mproduct")); - final int mscanOrdinal = super.getEffectorOrdinal(Bytes.encode(super.getEncoder(), "mscan")); + final int msumOrdinal = super.getEffectorOrdinal(Codec.encode("msum")); + final int mproductOrdinal = super.getEffectorOrdinal(Codec.encode("mproduct")); + final int mscanOrdinal = super.getEffectorOrdinal(Codec.encode("mscan")); final int[][] msumStateEffects = new int[nStates][]; final int[][] mproductStateEffects = new int[nStates][]; final int[][] mproductEndpoints = new int[nStates][2]; @@ -875,9 +895,9 @@ private void factor(final int[][][] transitionMatrix) { for (int input = 0; input < this.kernelMatrix.length; input++) { assert this.kernelMatrix[input].length == this.kernelMatrix[0].length; final IntsArray row = new IntsArray(this.kernelMatrix[input]); - HashSet equivalentClassOrdinals = rowEquivalenceMap.get(row); - if (equivalentClassOrdinals == null) { - equivalentClassOrdinals = new HashSet<>(16); + HashSet equivalentClassOrdinals = rowEquivalenceMap.computeIfAbsent( + row, absent -> new HashSet<>(16)); + if (equivalentClassOrdinals.isEmpty()) { rowEquivalenceMap.put(row, equivalentClassOrdinals); } equivalentClassOrdinals.add(input); diff --git a/src/com/characterforming/jrte/engine/ModelLoader.java b/src/com/characterforming/jrte/engine/ModelLoader.java index 4b6bc5d..40906fc 100644 --- a/src/com/characterforming/jrte/engine/ModelLoader.java +++ b/src/com/characterforming/jrte/engine/ModelLoader.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; +import java.nio.charset.CharacterCodingException; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicIntegerArray; @@ -54,7 +55,11 @@ public final class ModelLoader extends Model implements IModel { private ModelLoader(final File modelPath) throws ModelException { super(modelPath); - super.load(); + try { + super.load(); + } catch (CharacterCodingException e) { + throw new ModelException(e); + } int size = super.transducerOrdinalMap.size(); this.transducerAccessIndex = new AtomicIntegerArray(size); this.transducerObjectIndex = new AtomicReferenceArray<>(size); @@ -88,9 +93,10 @@ public boolean stream(final Bytes transducer, Signal prologue, InputStream in, O } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException | ClassNotFoundException e) { + String name = transducer.asString(); super.rteLogger.log(Level.SEVERE, e, () -> String.format( "Failed to instantiate target '%1$s' transducer '%2$s'", - super.getTargetClassname(), transducer.toString(super.getDecoder()))); + super.getTargetClassname(), name)); return false; } return this.stream(transducer, runTarget, prologue, in, out); @@ -211,8 +217,8 @@ private IEffector[] bindParameters(Transductor trex, IEffector[] runtimeEf @Override public void decompile(final String transducerName) - throws ModelException { - Transducer trex = this.loadTransducer(super.getTransducerOrdinal(Bytes.encode(encoder.reset(), transducerName))); + throws ModelException, CharacterCodingException { + Transducer trex = this.loadTransducer(super.getTransducerOrdinal(Codec.encode(transducerName))); int[] effectorVectors = trex.getEffectorVector(); int[] inputEquivalenceIndex = trex.getInputFilter(); long[] transitionMatrix = trex.getTransitionMatrix(); @@ -220,8 +226,8 @@ public void decompile(final String transducerName) Set> effectorOrdinalMap = super.getEffectorOrdinalMap().entrySet(); String[] effectorNames = new String[effectorOrdinalMap.size()]; for (Map.Entry entry : effectorOrdinalMap) { - effectorNames[entry.getValue()] = Bytes - .decode(super.getDecoder(), entry.getKey().bytes(), entry.getKey().getLength()).toString(); + effectorNames[entry.getValue()] = Codec.decode( + entry.getKey().bytes(), entry.getKey().getLength()); } System.out.printf("%s%n%nInput equivalents (equivalent: input...)%n%n", transducerName); for (int i = 0; i < inputEquivalentCount; i++) { @@ -265,7 +271,7 @@ public void decompile(final String transducerName) if (super.proxyEffectors[effectorOrdinal] instanceof BaseParameterizedEffector effector) { int parameterOrdinal = Transducer.parameter(effect); System.out.printf(" %s[", effectorNames[effectorOrdinal]); - System.out.printf(" %s ]", effector.showParameterTokens(super.getDecoder(), parameterOrdinal)); + System.out.printf(" %s ]", effector.showParameterTokens(parameterOrdinal)); } } else if (effect >= 0) { if (effect > 1) { @@ -281,7 +287,7 @@ public void decompile(final String transducerName) if (super.proxyEffectors[effectorOrdinal] instanceof BaseParameterizedEffector effector) { int parameterOrdinal = Transducer.parameter(effectorVectors[index++]); System.out.printf(" %s[", effectorNames[effectorOrdinal]); - System.out.printf(" %s ]", effector.showParameterTokens(this.getDecoder(), parameterOrdinal)); + System.out.printf(" %s ]", effector.showParameterTokens(parameterOrdinal)); } } } diff --git a/src/com/characterforming/jrte/engine/Token.java b/src/com/characterforming/jrte/engine/Token.java index 44c5a0f..fe1050d 100644 --- a/src/com/characterforming/jrte/engine/Token.java +++ b/src/com/characterforming/jrte/engine/Token.java @@ -19,7 +19,6 @@ */ package com.characterforming.jrte.engine; -import java.nio.charset.CharsetDecoder; import java.util.Arrays; import java.util.HashMap; @@ -144,11 +143,9 @@ public Type getType() { return this.type; } - @Override // @see com.characterforming.ribose.IToken#getName() - public String getName(CharsetDecoder decoder) { - return this.literal.getLength() != 1 || this.literal.bytes()[0] != 0xa - ? this.getLiteral().toString(decoder) - : "#a"; + @Override // @see com.characterforming.ribose.IToken#asString() + public String asString() { + return this.literal.asString(); } @Override // @see com.characterforming.ribose.IToken#getLiteralValue() diff --git a/src/com/characterforming/jrte/engine/Transductor.java b/src/com/characterforming/jrte/engine/Transductor.java index 53e5f60..7389d77 100644 --- a/src/com/characterforming/jrte/engine/Transductor.java +++ b/src/com/characterforming/jrte/engine/Transductor.java @@ -22,10 +22,7 @@ import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; import java.util.Arrays; import java.util.logging.Logger; @@ -118,8 +115,6 @@ public final class Transductor implements ITransductor, IOutput { private int errorInput; private final int signalLimit; private OutputStream outputStream; - private final CharsetDecoder decoder; - private final CharsetEncoder encoder; private final Logger rtcLogger; private final Logger rteLogger; private final ITransductor.Metrics metrics; @@ -131,8 +126,6 @@ public final class Transductor implements ITransductor, IOutput { this.model = null; this.loader = null; this.isProxy = true; - this.decoder = Base.newCharsetDecoder(); - this.encoder = Base.newCharsetEncoder(); this.transducerStack = null; this.selected = -1; this.value = null; @@ -153,8 +146,6 @@ public final class Transductor implements ITransductor, IOutput { super(); this.model = model; this.isProxy = false; - this.decoder = Base.newCharsetDecoder(); - this.encoder = Base.newCharsetEncoder(); this.loader = (ModelLoader)this.model; this.prologue = null; this.effectors = null; @@ -178,27 +169,31 @@ public final class Transductor implements ITransductor, IOutput { @Override // @see com.characterforming.ribose.ITarget#getEffectors() public IEffector[] getEffectors() throws TargetBindingException { - return new IEffector[] { - /* 0*/ new InlineEffector(this, "0"), - /* 1*/ new InlineEffector(this, "1"), - /* 2*/ new PasteEffector(this), - /* 3*/ new SelectEffector(this), - /* 4*/ new CopyEffector(this), - /* 5*/ new CutEffector(this), - /* 6*/ new ClearEffector(this), - /* 7*/ new CountEffector(this), - /* 8*/ new SignalEffector(this), - /* 9*/ new InEffector(this), - /*10*/ new OutEffector(this), - /*11*/ new InlineEffector(this, "mark"), - /*12*/ new InlineEffector(this, "reset"), - /*13*/ new StartEffector(this), - /*14*/ new PauseEffector(this), - /*15*/ new InlineEffector(this, "stop"), - /*16*/ new MsumEffector(this), - /*17*/ new MproductEffector(this), - /*18*/ new MscanEffector(this) - }; + try { + return new IEffector[] { + /* 0*/ new InlineEffector(this, "0"), + /* 1*/ new InlineEffector(this, "1"), + /* 2*/ new PasteEffector(this), + /* 3*/ new SelectEffector(this), + /* 4*/ new CopyEffector(this), + /* 5*/ new CutEffector(this), + /* 6*/ new ClearEffector(this), + /* 7*/ new CountEffector(this), + /* 8*/ new SignalEffector(this), + /* 9*/ new InEffector(this), + /*10*/ new OutEffector(this), + /*11*/ new InlineEffector(this, "mark"), + /*12*/ new InlineEffector(this, "reset"), + /*13*/ new StartEffector(this), + /*14*/ new PauseEffector(this), + /*15*/ new InlineEffector(this, "stop"), + /*16*/ new MsumEffector(this), + /*17*/ new MproductEffector(this), + /*18*/ new MscanEffector(this) + }; + } catch (CharacterCodingException e) { + throw new TargetBindingException(e); + } } @Override // @see com.characterforming.ribose.ITarget#getName() @@ -207,9 +202,9 @@ public String getName() { } @Override // @see com.characterforming.ribose.IOutput#getLocalizedFieldIndex(Bytes, Bytes) - public int getLocalizedFieldIndex(String transducerName, String fieldName) { - int transducerOrdinal = this.model.getTransducerOrdinal(Bytes.encode(this.encoder(), transducerName)); - int fieldOrdinal = this.model.getFieldOrdinal(Bytes.encode(this.encoder(), fieldName)); + public int getLocalizedFieldIndex(String transducerName, String fieldName) throws CharacterCodingException { + int transducerOrdinal = this.model.getTransducerOrdinal(Codec.encode(transducerName)); + int fieldOrdinal = this.model.getFieldOrdinal(Codec.encode(fieldName)); return this.model.getLocalField(transducerOrdinal, fieldOrdinal); } @@ -221,30 +216,21 @@ public int getLocalizedFieldndex() throws EffectorException { throw new EffectorException("Not valid for proxy transductor"); } - @Override // @see com.characterforming.ribose.IOutput#asBytes(int) - public byte[] asBytes(int fieldOrdinal) throws EffectorException { + @Override + public String asString(int fieldOrdinal) throws EffectorException, CharacterCodingException { if (!this.isProxy) { Value v = this.transducerStack.value(fieldOrdinal); - return Arrays.copyOf(v.value(), v.length()); - } else - throw new EffectorException("Not valid for proxy transductor"); + return Codec.decode(v.value(), v.length()); + } else { + throw new EffectorException("Not valid for proxy transductor"); + } } - @Override // @see com.characterforming.ribose.IOutput#asString(int) - public String asString(int fieldOrdinal) throws EffectorException { + @Override // @see com.characterforming.ribose.IOutput#asBytes(int) + public byte[] asBytes(int fieldOrdinal) throws EffectorException { if (!this.isProxy) { Value v = this.transducerStack.value(fieldOrdinal); - try { - return this.decoder().decode( - ByteBuffer.wrap(v.value(), 0, v.length())).toString(); - } catch (CharacterCodingException e) { - char[] chars = new char[v.length()]; - byte[] data = v.value(); - for (int i = 0; i < chars.length; i++) { - chars[i] = (char) (0xff & data[i]); - } - return new String(chars); - } + return Arrays.copyOf(v.value(), v.length()); } else throw new EffectorException("Not valid for proxy transductor"); } @@ -321,16 +307,6 @@ public Logger getRteLogger() { return this.rteLogger; } - @Override // @see com.characterforming.ribose.IOutput#decoder() - public CharsetDecoder decoder() { - return this.decoder.reset(); - } - - @Override // @see com.characterforming.ribose.IOutput#encoder() - public CharsetEncoder encoder() { - return this.encoder.reset(); - } - @Override // @see com.characterforming.ribose.ITransductor#recycle() public byte[] recycle(byte[] bytes) { return this.inputStack.recycle(bytes); @@ -769,7 +745,7 @@ private String getErrorInput(int last, int state) { } private final class InlineEffector extends BaseEffector { - private InlineEffector(final Transductor transductor, final String name) { + private InlineEffector(final Transductor transductor, final String name) throws CharacterCodingException { super(transductor, name); } @@ -780,7 +756,7 @@ public int invoke() throws EffectorException { } private final class PasteEffector extends BaseInputOutputEffector { - private PasteEffector(final Transductor transductor) { + private PasteEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "paste"); } @@ -803,7 +779,7 @@ public int invoke(final int parameterIndex) throws EffectorException { value.paste(bytes, bytes.length); } else { throw new EffectorException(String.format("Invalid token `%1$s` for effector '%2$s'", - token.getName(super.decoder()), super.getName())); + token.asString(), super.getName())); } } } @@ -812,7 +788,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class SelectEffector extends BaseFieldEffector { - private SelectEffector(final Transductor transductor) { + private SelectEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "select"); } @@ -836,7 +812,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class CopyEffector extends BaseFieldEffector { - private CopyEffector(final Transductor transductor) { + private CopyEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "copy"); } @@ -856,7 +832,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class CutEffector extends BaseFieldEffector { - private CutEffector(final Transductor transductor) { + private CutEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "cut"); } @@ -876,7 +852,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class ClearEffector extends BaseFieldEffector { - private ClearEffector(final Transductor transductor) { + private ClearEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "clear"); } @@ -901,7 +877,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class SignalEffector extends BaseParameterizedEffector { - private SignalEffector(final Transductor transductor) { + private SignalEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "signal"); } @@ -928,13 +904,20 @@ public Integer compileParameter(final IToken[] parameterList) throws TargetBindi if (token.getType() == IToken.Type.SIGNAL) { int ordinal = token.getOrdinal(); if (ordinal < 0) { - throw new TargetBindingException(String.format("Null signal reference for signal effector: %s", - token.getName(super.decoder()))); + String signame; + try { + byte[] sigbytes = token.getLiteral().bytes(); + signame = Codec.decode(sigbytes); + } catch (CharacterCodingException e) { + signame = ""; + } + throw new TargetBindingException(String.format( + "Unkown signal reference for signal effector: %s", signame)); } return ordinal; } else { throw new TargetBindingException(String.format("Invalid signal reference `%s` for signal effector, requires type indicator ('%c') before the transducer name", - token.getName(super.decoder()), IToken.SIGNAL_TYPE)); + token.asString(), IToken.SIGNAL_TYPE)); } } else { throw new TargetBindingException(String.format("Unknown IToken implementation class '%1$s'", @@ -944,7 +927,7 @@ public Integer compileParameter(final IToken[] parameterList) throws TargetBindi } private final class InEffector extends BaseInputOutputEffector { - private InEffector(final Transductor transductor) { + private InEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "in"); } @@ -973,7 +956,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class OutEffector extends BaseInputOutputEffector { - private OutEffector(final Transductor transductor) { + private OutEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "out"); } @@ -1007,7 +990,7 @@ public int invoke(final int parameterIndex) throws EffectorException { } private final class CountEffector extends BaseParameterizedEffector { - private CountEffector(final Transductor transductor) { + private CountEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "count"); } @@ -1046,9 +1029,8 @@ public int[] compileParameter(final IToken[] parameterList) throws TargetBinding byte[] v = parameterList[0].getLiteral().bytes(); count = Base.decodeInt(v, v.length); } else { - byte[] v = parameterList[0].getLiteral().bytes(); - throw new TargetBindingException(String.format("%1$s.%2$s[]: invalid field|counter '%3$%s' for count effector", - super.getTarget().getName(), super.getName(), Bytes.decode(super.decoder(), v, v.length))); + throw new TargetBindingException(String.format("%1$s.%2$s[]: invalid field|counter for count effector", + super.getTarget().getName(), super.getName())); } if (parameterList[1].getType() == IToken.Type.SIGNAL) { int signalOrdinal = parameterList[1].getOrdinal(); @@ -1056,13 +1038,13 @@ public int[] compileParameter(final IToken[] parameterList) throws TargetBinding return new int[] { count, signalOrdinal }; } else { throw new TargetBindingException(String.format("%1$s.%2$s[]: invalid signal '%3$%s' for count effector", - super.getTarget().getName(), super.getName(), parameterList[1].getName(this.decoder()))); + super.getTarget().getName(), super.getName(), parameterList[1].asString())); } } } private final class StartEffector extends BaseParameterizedEffector { - private StartEffector(final Transductor transductor) { + private StartEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "start"); } @@ -1090,20 +1072,22 @@ public Integer compileParameter(final IToken[] parameterTokens) throws TargetBin @Override public int invoke(final int parameterIndex) throws EffectorException { + int transducerOrdinal = super.getParameter(parameterIndex); try { - transducerStack.push(loader.loadTransducer(super.getParameter(parameterIndex))); + transducerStack.push(loader.loadTransducer(transducerOrdinal)); transducerStack.peek().selected = Model.ANONYMOUS_FIELD_ORDINAL; } catch (final ModelException e) { - byte[] bytes = model.getTransducerName(super.getParameter(parameterIndex)); - throw new EffectorException(String.format("The start effector failed to load %1$s", - Bytes.decode(super.decoder(), bytes, bytes.length)), e); + throw new EffectorException(String.format( + "The start effector failed to load transducer with ordinal number %1$d", + transducerOrdinal), e + ); } return IEffector.RTX_START; } } private final class PauseEffector extends BaseEffector { - private PauseEffector(final Transductor transductor) { + private PauseEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "pause"); } @@ -1114,7 +1098,7 @@ public int invoke() throws EffectorException { } private final class MsumEffector extends BaseParameterizedEffector { - private MsumEffector(final Transductor transductor) { + private MsumEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "msum"); } @@ -1154,7 +1138,7 @@ public long[] compileParameter(final IToken[] parameterList) throws TargetBindin } @Override - public String showParameterTokens(CharsetDecoder decoder, int parameterIndex) { + public String showParameterTokens(int parameterIndex) { long[] sum = super.getParameter(parameterIndex); StringBuilder sb = new StringBuilder(); int endBit = 0, startBit = -1; @@ -1193,7 +1177,7 @@ private void printRange(StringBuilder sb, int startBit, int endBit) { } private final class MproductEffector extends BaseParameterizedEffector { - private MproductEffector(final Transductor transductor) { + private MproductEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "mproduct"); } @@ -1230,7 +1214,7 @@ public byte[] compileParameter(final IToken[] parameterList) throws TargetBindin } @Override - public String showParameterTokens(CharsetDecoder decoder, int parameterIndex) { + public String showParameterTokens(int parameterIndex) { byte[] product = super.getParameter(parameterIndex); StringBuilder sb = new StringBuilder(); for (int j = 0; j < product.length; j++) { @@ -1243,7 +1227,7 @@ public String showParameterTokens(CharsetDecoder decoder, int parameterIndex) { } private final class MscanEffector extends BaseParameterizedEffector { - private MscanEffector(final Transductor transductor) { + private MscanEffector(final Transductor transductor) throws CharacterCodingException { super(transductor, "mscan"); } @@ -1278,7 +1262,7 @@ public Integer compileParameter(final IToken[] parameterList) throws TargetBindi } @Override - public String showParameterTokens(CharsetDecoder decoder, int parameterIndex) { + public String showParameterTokens(int parameterIndex) { int scanbyte = super.getParameter(parameterIndex); return 32 < scanbyte && 127 > scanbyte ? String.format(" %c", (char)scanbyte) diff --git a/src/com/characterforming/jrte/test/FileRunner.java b/src/com/characterforming/jrte/test/FileRunner.java index 6a01042..5aaa45f 100644 --- a/src/com/characterforming/jrte/test/FileRunner.java +++ b/src/com/characterforming/jrte/test/FileRunner.java @@ -24,17 +24,13 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CoderResult; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.characterforming.jrte.engine.Base; +import com.characterforming.jrte.engine.Codec; import com.characterforming.ribose.IModel; import com.characterforming.ribose.ITransductor; import com.characterforming.ribose.ITransductor.Metrics; @@ -83,14 +79,12 @@ public static void main(final String[] args) { int exitCode = 1; Base.startLogging(); final Logger rteLogger = Base.getRuntimeLogger(); - final CharsetDecoder decoder = Base.newCharsetDecoder(); - final CharsetEncoder encoder = Base.newCharsetEncoder(); byte[] bbuf = new byte[(int)blen]; - CharBuffer charInput = null; + String charInput = null; try (final FileInputStream isr = new FileInputStream(f)) { blen = isr.read(bbuf, 0, (int)blen); assert blen == bbuf.length; - charInput = decoder.reset().decode(ByteBuffer.wrap(bbuf, 0, bbuf.length)); + charInput = Codec.decode(bbuf); } catch (Exception e) { System.out.println("Runtime exception thrown."); rteLogger.log(Level.SEVERE, "Runtime failed, exception thrown.", e); @@ -110,7 +104,7 @@ public static void main(final String[] args) { assert trex.status().isStopped(); if (trex.push(bbuf, (int)blen).status().isWaiting() && (!nil || (trex.signal(Signal.NIL).status().isWaiting())) - && (trex.start(Bytes.encode(encoder, transducerName)).status().isRunnable())) { + && (trex.start(Codec.encode(transducerName)).status().isRunnable())) { t0 = System.nanoTime(); do { trex.run(); @@ -142,7 +136,7 @@ public static void main(final String[] args) { final FileInputStream isr = new FileInputStream(f); final BufferedOutputStream osw = new BufferedOutputStream(System.out, Base.getOutBufferSize()) ) { - ribose.stream(Bytes.encode(encoder, transducerName), nil ? Signal.NIL : Signal.NONE, isr, osw); + ribose.stream(Codec.encode(transducerName), nil ? Signal.NIL : Signal.NONE, isr, osw); } catch (Exception e) { System.out.println("Runtime exception thrown."); rteLogger.log(Level.SEVERE, "Runtime failed, exception thrown.", e); @@ -187,35 +181,19 @@ public static void main(final String[] args) { System.out.println(String.format(" : %8.3f mb/s", mbps)); } else { int count = 0; - charInput.rewind(); - byte[] bytes = new byte[Base.getOutBufferSize()]; - ByteBuffer bbuffer = ByteBuffer.wrap(bytes); - CharBuffer cbuffer = CharBuffer.allocate(bytes.length); + final byte[] delimiters = new byte[] {'|','\n'}; Matcher matcher = pattern.matcher(charInput); while (matcher.find()) { int k = matcher.groupCount(); for (int j = 1; j <= k; j++) { - String match = matcher.group(j); - if (match == null) { - match = ""; - } - if (cbuffer.remaining() < (match.length() + 1)) { - CoderResult code = encoder.encode(cbuffer.flip(), bbuffer, false); - assert code.isUnderflow(); - System.out.write(bbuffer.array(), 0, bbuffer.position()); - bbuffer.clear(); cbuffer.clear(); - } - cbuffer.append(match).append(j < k ? '|' : '\n'); + String group = matcher.group(j); + System.out.write(group != null ? Codec.encode(group).bytes() : Bytes.EMPTY_BYTES); + System.out.write(delimiters[j < k ? 0 : 1]); } if (k > 0) { count++; } } - if (cbuffer.position() > 0) { - CoderResult code = encoder.encode(cbuffer.flip(), bbuffer, true); - assert code.isUnderflow(); - System.out.write(bytes, 0, bbuffer.position()); - } System.out.flush(); assert count > 0; } diff --git a/src/com/characterforming/jrte/test/TestRunner.java b/src/com/characterforming/jrte/test/TestRunner.java index 95308c2..4687a59 100644 --- a/src/com/characterforming/jrte/test/TestRunner.java +++ b/src/com/characterforming/jrte/test/TestRunner.java @@ -21,13 +21,13 @@ import java.io.File; import java.nio.CharBuffer; -import java.nio.charset.CharsetEncoder; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.characterforming.jrte.engine.Base; +import com.characterforming.jrte.engine.Codec; import com.characterforming.ribose.IModel; import com.characterforming.ribose.ITransductor; import com.characterforming.ribose.ITransductor.Status; @@ -63,13 +63,12 @@ public static void main(final String[] args) { String[] tests = new String[] { "NilSpeedTest", "PasteSpeedTest", "PasteCutTest", "SelectPasteTest", "PasteCountTest", "CounterTest", "NilPauseTest", "PastePauseTest", "StackTest" }; - final CharsetEncoder encoder = Base.newCharsetEncoder(); try (final IModel ribose = IModel.loadRiboseModel(new File(modelPath))) { final ITransductor trex = ribose.transductor(new TestTarget()); for (final String test : tests) { long t0 = 0, t1 = 0, t2 = 0; System.out.format("%20s: ", test); - Bytes transducer = Bytes.encode(encoder, test); + Bytes transducer = Codec.encode(test); for (int i = 0; i < 20; i++) { assert trex.status() == Status.STOPPED; trex.push(abytes, abytes.length).signal(Signal.NIL); diff --git a/src/com/characterforming/jrte/test/TestTarget.java b/src/com/characterforming/jrte/test/TestTarget.java index a86612b..a15cdbf 100644 --- a/src/com/characterforming/jrte/test/TestTarget.java +++ b/src/com/characterforming/jrte/test/TestTarget.java @@ -1,5 +1,8 @@ package com.characterforming.jrte.test; +import java.nio.charset.CharacterCodingException; + +import com.characterforming.jrte.engine.Codec; import com.characterforming.ribose.IEffector; import com.characterforming.ribose.IOutput; import com.characterforming.ribose.ITarget; @@ -21,11 +24,15 @@ public TestTarget() { @Override public IEffector[] getEffectors() throws TargetBindingException { - return new IEffector[] { - new IntegerValueEffector(this), - new RealValueEffector(this), - new StringValueEffector(this) - }; + try { + return new IEffector[] { + new IntegerValueEffector(this), + new RealValueEffector(this), + new StringValueEffector(this) + }; + } catch (CharacterCodingException e) { + throw new TargetBindingException(e); + } } @Override // ITarget#getName() @@ -34,7 +41,7 @@ public String getName() { } private class IntegerValueEffector extends BaseEffector { - IntegerValueEffector(TestTarget target) { + IntegerValueEffector(TestTarget target) throws CharacterCodingException { super(target, "integer"); } @@ -42,8 +49,8 @@ private class IntegerValueEffector extends BaseEffector { public int invoke() throws EffectorException { long integer = 0; try { - integer = Long.parseLong(super.output.asString(0)); - } catch (NumberFormatException e) { + integer = Long.parseLong(Codec.decode(super.output.asBytes(0))); + } catch (NumberFormatException | CharacterCodingException e) { return super.output.signal(fail); } return super.output.signal(integer == super.output.asInteger(0) ? pass : fail); @@ -51,7 +58,7 @@ public int invoke() throws EffectorException { } private class RealValueEffector extends BaseEffector { - RealValueEffector(TestTarget target) { + RealValueEffector(TestTarget target) throws CharacterCodingException { super(target, "real"); } @@ -59,8 +66,8 @@ private class RealValueEffector extends BaseEffector { public int invoke() throws EffectorException { double real = 0.0; try { - real = Double.parseDouble(super.output.asString(0)); - } catch (NumberFormatException e) { + real = Double.parseDouble(Codec.decode(super.output.asBytes(0))); + } catch (NumberFormatException | CharacterCodingException e) { return super.output.signal(fail); } return super.output.signal(real == super.output.asReal(0) ? pass : fail); @@ -68,7 +75,7 @@ public int invoke() throws EffectorException { } private class StringValueEffector extends BaseEffector { - StringValueEffector(TestTarget target) { + StringValueEffector(TestTarget target) throws CharacterCodingException { super(target, "string"); } @@ -79,7 +86,13 @@ public void setOutput(IOutput output) throws EffectorException { @Override public int invoke() throws EffectorException { - String field = super.output.asString(0); + byte[] bytes = super.output.asBytes(0); + String field; + try { + field = Codec.decode(bytes); + } catch (CharacterCodingException e) { + field = ""; + } return super.output.signal(field.equals("pi") || field.equals("-pi") ? pass : fail); } } diff --git a/src/com/characterforming/ribose/IEffector.java b/src/com/characterforming/ribose/IEffector.java index 44f9444..5e569af 100644 --- a/src/com/characterforming/ribose/IEffector.java +++ b/src/com/characterforming/ribose/IEffector.java @@ -84,7 +84,8 @@ public interface IEffector { * @return user-defined effectors should return {@code IEffector.RTX_NONE} * @throws EffectorException if things don't work out */ - int invoke() throws EffectorException; + int invoke() + throws EffectorException; /** * Receive an {@link IOutput} view of the transductor that the effector target @@ -95,7 +96,8 @@ public interface IEffector { * @throws EffectorException if field names can't be resolved * @see IOutput#getLocalizedFieldIndex(String, String) */ - void setOutput(IOutput output) throws EffectorException; + void setOutput(IOutput output) + throws EffectorException; /** * Returns the target that expresses the effector. @@ -129,10 +131,10 @@ default boolean equivalent(final IEffector other) { /** * Called for proxy effectors after parameter compilation is - * complete. This will null out the target, output, decoder and - * encoder fields in the {@code BaseEffector} superclass. This - * allows the proxy model and transducer to be garbage collected - * after all effector parameters have been compiled. Subclasses + * complete. This will null out the target and output fields + * in the {@code BaseEffector} superclass. This allows the + * proxy model and transducer to be garbage collected after + * all effector parameters have been compiled. Subclasses * may override this method to dispose of additional resources * as well, but must also call {@code super.passivate()} in * the overriding method. diff --git a/src/com/characterforming/ribose/IModel.java b/src/com/characterforming/ribose/IModel.java index 7729db2..6d0296f 100644 --- a/src/com/characterforming/ribose/IModel.java +++ b/src/com/characterforming/ribose/IModel.java @@ -24,7 +24,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; +import java.nio.charset.CharacterCodingException; +import com.characterforming.jrte.engine.Codec; import com.characterforming.jrte.engine.ModelCompiler; import com.characterforming.jrte.engine.ModelLoader; import com.characterforming.ribose.base.Bytes; @@ -46,7 +48,11 @@ * multithreaded contexts. In all other respects, ribose objects are not threadsafe. * Runtime {@code IModel} instances are {@link AutoCloseable} but clients must * call {@link #close()} for models that are instantiated outside a try-with-resources - * block. + * block. Character set decoder and encoder are held in Codec object attached as a + * {@link ThreadLocal} to threads that require them and detached when the model is + * closed by the thread. Threads that use ribose APIs but are not involved + * with closing a model must call {@link #detach()} explicitly when they no longer + * require ribose APIs. * * @author Kim Briggs * @@ -150,9 +156,10 @@ boolean stream(Bytes transducer, ITarget target, Signal prologue, InputStream in * * @param transducerName the transducer name as aUnicode string * @throws ModelException if things don't work out + * @throws CharacterCodingException if encoder fails */ void decompile(String transducerName) - throws ModelException; + throws ModelException, CharacterCodingException; /** * Print the model map to an output stream @@ -172,11 +179,18 @@ boolean map(PrintStream mapWriter) String getTargetClassname(); /** - * Close the runtime model and file. + * Close the runtime model and file and detach ThreadLocal variables. * * @throws ModelException if things don't work out */ @Override void close() throws ModelException; + + /** + * Detach thread local variables from the calling thread. + */ + static void detach() { + Codec.detach(); + } } diff --git a/src/com/characterforming/ribose/IOutput.java b/src/com/characterforming/ribose/IOutput.java index 3993d26..79a9719 100755 --- a/src/com/characterforming/ribose/IOutput.java +++ b/src/com/characterforming/ribose/IOutput.java @@ -20,8 +20,7 @@ package com.characterforming.ribose; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; +import java.nio.charset.CharacterCodingException; import java.util.logging.Logger; import com.characterforming.ribose.base.EffectorException; @@ -100,8 +99,10 @@ public interface IOutput { * @param fieldName the name of the field (`~` prefix in lead byte) * @return the localized index of the field in the current transducer stack frame * @throws EffectorException if things don't work out + * @throws CharacterCodingException if encoder fails */ - int getLocalizedFieldIndex(String transducerName, String fieldName) throws EffectorException; + int getLocalizedFieldIndex(String transducerName, String fieldName) + throws EffectorException, CharacterCodingException; /** * Get the ordinal number for the current selected field. Not valid for proxy effectors. @@ -110,34 +111,39 @@ public interface IOutput { * @return the localized index of the selected field in the current transducer stack frame * @throws EffectorException if called on a proxy transductor */ - int getLocalizedFieldndex() throws EffectorException; + int getLocalizedFieldndex() + throws EffectorException; /** * Get current field value as integer value. * * @param fieldOrdinal the field ordinal - * @return the integer value decoded from the field contents + * @return the String value decoded from the field contents * @throws EffectorException if called on a proxy transductor + * @throws CharacterCodingException if decoding fails */ - long asInteger(int fieldOrdinal) throws EffectorException; + String asString(int fieldOrdinal) + throws EffectorException, CharacterCodingException; /** - * Get current field value as real value. + * Get current field value as integer value. * * @param fieldOrdinal the field ordinal - * @return the real value decoded from the field contents + * @return the integer value decoded from the field contents * @throws EffectorException if called on a proxy transductor */ - double asReal(int fieldOrdinal) throws EffectorException; + long asInteger(int fieldOrdinal) + throws EffectorException; /** - * Get current field value as integer value. + * Get current field value as real value. * * @param fieldOrdinal the field ordinal - * @return the string decoded from the field contents + * @return the real value decoded from the field contents * @throws EffectorException if called on a proxy transductor */ - String asString(int fieldOrdinal) throws EffectorException; + double asReal(int fieldOrdinal) + throws EffectorException; /** * Get current field value as integer value. @@ -146,7 +152,8 @@ public interface IOutput { * @return the field contents * @throws EffectorException if called on a proxy transductor */ - byte[] asBytes(int fieldOrdinal) throws EffectorException; + byte[] asBytes(int fieldOrdinal) + throws EffectorException; /** * Encode a signal in an {@code effector.invoke()} return value. The signal will be @@ -169,21 +176,8 @@ public interface IOutput { * @throws EffectorException if {@code signalOrdinal} is out of range * @see Signal#signal() */ - int signal(int signalOrdinal) throws EffectorException; - - /** - * Get and reset the decoder bound to the transductor - * - * @return the transductor's decoder - */ - CharsetDecoder decoder(); - - /** - * Get and reset the encoder bound to the transductor - * - * @return the transductor's encoder - */ - CharsetEncoder encoder(); + int signal(int signalOrdinal) + throws EffectorException; /** * Get the ribose compiler Logger instance diff --git a/src/com/characterforming/ribose/IParameterizedEffector.java b/src/com/characterforming/ribose/IParameterizedEffector.java index 28fa4d3..a33fccd 100644 --- a/src/com/characterforming/ribose/IParameterizedEffector.java +++ b/src/com/characterforming/ribose/IParameterizedEffector.java @@ -20,8 +20,6 @@ package com.characterforming.ribose; -import java.nio.charset.CharsetDecoder; - import com.characterforming.ribose.base.BaseParameterizedEffector; import com.characterforming.ribose.base.EffectorException; import com.characterforming.ribose.base.Signal; @@ -55,7 +53,7 @@ * effectors and decompiling transducers. Proxy efectors lose access to their * proxy target instance after they are passivated but retain their compiled * P[] instances. Proxy effectors may receive calls to {@link - * #showParameterType()} and {@link #showParameterTokens(CharsetDecoder, int)} + * #showParameterType()} and {@link #showParameterTokens(int)} * from the model decompiler; these methods are implemented in {@link * BaseParameterizedEffector} but may be overridden by subclasses. The * {@link IEffector#invoke()} and {@link #invoke(int)} methods are never called @@ -69,7 +67,7 @@ * } * ... * public SimpleDateFormat compileParameter(IToken[] parameterTokens) { - * String format = parameterTokens[0].getLiteral().toString(super.getDecoder()); + * String format = parameterTokens[0].asString(); * return new SimpleDateFormat(format); * } * ... @@ -110,7 +108,8 @@ public interface IParameterizedEffector extends IEffector< * @return user-defined effectors should return 0 (RTX_SI) * @throws EffectorException if things don't work out */ - int invoke(int parameterIndex) throws EffectorException; + int invoke(int parameterIndex) + throws EffectorException; /** * Allocate an array (P[]) to hold precompiled parameter objects @@ -129,7 +128,8 @@ public interface IParameterizedEffector extends IEffector< * @return the compiled parameter value object * @throws TargetBindingException if things don't work out */ - P compileParameter(IToken[] parameterTokens) throws TargetBindingException; + P compileParameter(IToken[] parameterTokens) + throws TargetBindingException; /** * Return the type of the effector's parameter object, to support decompilation @@ -146,9 +146,8 @@ public interface IParameterizedEffector extends IEffector< * decompilation. This is implemented in {@link BaseParameterizedEffector} * but may be overriden by subclasses. * - * @param decoder the decoder to use * @param parameterIndex the parameter index * @return a printable string of space-delimited raw parameter tokens */ - String showParameterTokens(CharsetDecoder decoder, int parameterIndex); + String showParameterTokens(int parameterIndex); } diff --git a/src/com/characterforming/ribose/ITarget.java b/src/com/characterforming/ribose/ITarget.java index ce67093..8594234 100644 --- a/src/com/characterforming/ribose/ITarget.java +++ b/src/com/characterforming/ribose/ITarget.java @@ -119,5 +119,6 @@ public interface ITarget { * @return an array of IEffector instances bound to the target instance * @throws TargetBindingException if things don't work out */ - IEffector[] getEffectors() throws TargetBindingException; + IEffector[] getEffectors() + throws TargetBindingException; } diff --git a/src/com/characterforming/ribose/IToken.java b/src/com/characterforming/ribose/IToken.java index 8028019..a029152 100644 --- a/src/com/characterforming/ribose/IToken.java +++ b/src/com/characterforming/ribose/IToken.java @@ -20,8 +20,6 @@ package com.characterforming.ribose; -import java.nio.charset.CharsetDecoder; - import com.characterforming.ribose.base.Bytes; /** @@ -103,12 +101,13 @@ String getName() { Type getType(); /** - * Get the name of the token. + * Get the a string representing the token. This will obtained by decoding the + * token literal value as a UTF-8 byte sequence, or converting all bytes to + * /xHH representation if decoding fails. * - * @param decoder the decoder to use for the name - * @return the name of this token + * @return a string representation of this token */ - String getName(CharsetDecoder decoder); + String asString(); /** * Get the name of the type, eg "signal", "field" or "transducer". diff --git a/src/com/characterforming/ribose/ITransductor.java b/src/com/characterforming/ribose/ITransductor.java index bbdc915..293f9a8 100644 --- a/src/com/characterforming/ribose/ITransductor.java +++ b/src/com/characterforming/ribose/ITransductor.java @@ -316,7 +316,8 @@ public void update(Metrics accumulator) { * @return this ITransductor * @throws ModelException if things don't work out */ - ITransductor start(Bytes transducer) throws ModelException; + ITransductor start(Bytes transducer) + throws ModelException; /** * Run the transduction with current input until the input or transduction @@ -344,7 +345,8 @@ public void update(Metrics accumulator) { * @see #recycle(byte[]) * @see #status() */ - ITransductor run() throws EffectorException, DomainErrorException; + ITransductor run() + throws EffectorException, DomainErrorException; /** * Return a new or unmarked byte[] buffer if the previous buffer ({@code bytes}) @@ -379,5 +381,6 @@ public void update(Metrics accumulator) { * @throws RiboseException if transductor is proxy for parameter compilation * @see #status() */ - ITransductor stop() throws RiboseException; + ITransductor stop() + throws RiboseException; } \ No newline at end of file diff --git a/src/com/characterforming/ribose/Ribose.java b/src/com/characterforming/ribose/Ribose.java index fc3d6c9..936c211 100755 --- a/src/com/characterforming/ribose/Ribose.java +++ b/src/com/characterforming/ribose/Ribose.java @@ -27,13 +27,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.charset.CharsetEncoder; +import java.nio.charset.CharacterCodingException; import java.util.Arrays; import java.util.logging.Level; import java.util.logging.Logger; import com.characterforming.jrte.engine.Base; -import com.characterforming.ribose.base.Bytes; +import com.characterforming.jrte.engine.Codec; import com.characterforming.ribose.base.ModelException; import com.characterforming.ribose.base.RiboseException; import com.characterforming.ribose.base.Signal; @@ -183,7 +183,8 @@ public static void main(final String[] args) { System.exit(fail ? 1 : 0); } - private static boolean execCompile(final String[] args) throws SecurityException { + private static boolean execCompile(final String[] args) + throws SecurityException { Base.startLogging(); File ginrAutomataDirectory = null; File riboseModelFile = null; @@ -249,7 +250,8 @@ private static boolean execCompile(final String[] args) throws SecurityException return compiled; } - private static boolean execRun(final String[] args) throws SecurityException { + private static boolean execRun(final String[] args) + throws SecurityException { int argc = args.length; boolean nil = argc > 0 && args[0].compareTo("--nil") == 0; int arg = nil ? 1 : 0; @@ -282,7 +284,6 @@ private static boolean execRun(final String[] args) throws SecurityException { final String inputPath = arg < argc ? args[arg++] : "-"; final String output = arg < argc ? args[arg++] : null; final Logger rteLogger = Base.getRuntimeLogger(); - final CharsetEncoder encoder = Base.newCharsetEncoder(); final File input = inputPath.charAt(0) == '-' ? null : new File(inputPath); boolean run = true; if (input != null && !input.exists()) { @@ -303,8 +304,7 @@ private static boolean execRun(final String[] args) throws SecurityException { ); ) { run = ribose.stream( - Bytes.encode(encoder, transducerName), - nil ? Signal.NIL : Signal.NONE, streamIn, streamOut + Codec.encode(transducerName), nil ? Signal.NIL : Signal.NONE, streamIn, streamOut ); streamOut.flush(); } catch (final IOException | ModelException | RiboseException e) { @@ -341,7 +341,7 @@ private static boolean execDecompile(final String[] args) { try (IModel model = IModel.loadRiboseModel(modelFile)) { model.decompile(transducerName); decompiled = true; - } catch (ModelException e) { + } catch (ModelException | CharacterCodingException e) { final String format = "Failed to decompile %1$s"; rteLogger.log(Level.SEVERE, e, () -> String.format(format, transducerName)); diff --git a/src/com/characterforming/ribose/base/BaseEffector.java b/src/com/characterforming/ribose/base/BaseEffector.java index d4f4224..f415660 100644 --- a/src/com/characterforming/ribose/base/BaseEffector.java +++ b/src/com/characterforming/ribose/base/BaseEffector.java @@ -20,10 +20,8 @@ package com.characterforming.ribose.base; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; - -import com.characterforming.jrte.engine.Base; +import java.nio.charset.CharacterCodingException; +import com.characterforming.jrte.engine.Codec; import com.characterforming.ribose.IEffector; import com.characterforming.ribose.IOutput; import com.characterforming.ribose.ITarget; @@ -50,9 +48,11 @@ public abstract class BaseEffector implements IEffector { * * @param target the target that binds the effector * @param name the effector name + * @throws CharacterCodingException if encoder fails */ - protected BaseEffector(final T target, final String name) { - this.name = Bytes.encode(Base.newCharsetEncoder(), name); + protected BaseEffector(final T target, final String name) + throws CharacterCodingException { + this.name = Codec.encode(name); this.target = target; this.output = null; } @@ -61,7 +61,8 @@ protected BaseEffector(final T target, final String name) { public abstract int invoke() throws EffectorException; @Override // @see com.characterforming.ribose.base.IEffector#setOutput(IOutput) - public void setOutput(IOutput output) throws EffectorException { + public void setOutput(IOutput output) + throws EffectorException { this.output = output; } @@ -77,7 +78,7 @@ public final T getTarget() { @Override // @see java.lang.Object#toString() public String toString() { - return this.name.toString(this.decoder()); + return this.name.asString(); } @Override // com.characterforming.ribose.IEffector#passivate() @@ -85,22 +86,4 @@ public void passivate() { this.target = null; this.output = null; } - - /** - * Lazy instantiation for charset decoder - * - * @return a decoder instance - */ - protected CharsetDecoder decoder() { - return this.output.decoder(); - } - - /** - * Lazy instantiation for charset encoder - * - * @return a encoder instance - */ - protected CharsetEncoder encoder() { - return this.output.encoder(); - } } diff --git a/src/com/characterforming/ribose/base/BaseParameterizedEffector.java b/src/com/characterforming/ribose/base/BaseParameterizedEffector.java index afacf7a..ca0eb5b 100644 --- a/src/com/characterforming/ribose/base/BaseParameterizedEffector.java +++ b/src/com/characterforming/ribose/base/BaseParameterizedEffector.java @@ -20,7 +20,7 @@ package com.characterforming.ribose.base; -import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharacterCodingException; import java.util.List; import com.characterforming.ribose.IParameterizedEffector; @@ -37,7 +37,7 @@ *
  • {@link IParameterizedEffector#allocateParameters(int)}
    • *
  • {@link IParameterizedEffector#compileParameter(IToken[])}
    • * - * Default {@link showParameterType()} and {@link showParameterTokens(CharsetDecoder, int)} + * Default {@link showParameterType()} and {@link showParameterTokens(int)} * methods are implemented here but subclasses may override these if desired. * Otherwise, public methods not exposed in the {@link IParameterizedEffector} interface * are for internal use only. These methods implement the parameter compilation and binding @@ -59,8 +59,10 @@ public abstract class BaseParameterizedEffector extends Ba * * @param target the target for the effector * @param name the effector name as referenced from ginr transducers + * @throws CharacterCodingException if encoder fails */ - protected BaseParameterizedEffector(final T target, final String name) { + protected BaseParameterizedEffector(final T target, final String name) + throws CharacterCodingException { super(target, name); } @@ -81,11 +83,11 @@ public String showParameterType() { } @Override - public String showParameterTokens(CharsetDecoder decoder, int parameterIndex) { + public String showParameterTokens(int parameterIndex) { StringBuilder sb = new StringBuilder(256); for (IToken token : this.tokens[parameterIndex]) { sb.append(sb.length() == 0 ? "`" : " `") - .append(token.getLiteral().toString(decoder)) + .append(token.getLiteral().asString()) .append("`"); } return sb.toString(); diff --git a/src/com/characterforming/ribose/base/Bytes.java b/src/com/characterforming/ribose/base/Bytes.java index 210539b..704e110 100644 --- a/src/com/characterforming/ribose/base/Bytes.java +++ b/src/com/characterforming/ribose/base/Bytes.java @@ -20,14 +20,10 @@ package com.characterforming.ribose.base; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; import java.nio.charset.CharacterCodingException; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CharsetEncoder; import java.util.Arrays; -import com.characterforming.jrte.engine.Base; +import com.characterforming.jrte.engine.Codec; /** * Wraps an immutable array of bytes. Ribose transductions operate in the {@code byte} @@ -37,10 +33,11 @@ */ public final class Bytes { /** A singleton empty byte array */ - public static final byte[] EMPTY_BYTES = new byte[] { }; + public static final byte[] EMPTY_BYTES = new byte[] {}; + /** A singleton Bytes instance wrapping an empty byte array */ + public static final Bytes EMPTY = new Bytes(Bytes.EMPTY_BYTES); private static final char[] HEX = "0123456789ABCDEF".toCharArray(); - private static final CharBuffer DECODER_ERROR = CharBuffer.wrap(new char[] { '?', '?', '?' }); private final byte[] data; private int hash; @@ -70,58 +67,6 @@ public Bytes(final byte[] from, int offset, int length) { this.hash = 0; } - /** - * Decode UTF-8 bytes to a String. - * - * @param decoder the decoder to use - * @param bytes the bytes to decode - * @param length the number of bytes to decode - * @return a CharBuffer containing the decoded text - */ - public static CharBuffer decode(CharsetDecoder decoder, final byte[] bytes, final int length) { - assert 0 <= length && length <= bytes.length; - int size = Math.max(Math.min(length, bytes.length), 0); - try { - return decoder.reset().decode(ByteBuffer.wrap(bytes, 0, size)); - } catch (CharacterCodingException e) { - assert false; - System.err.printf("Bytes.decode(CharsetDecoder, byte[], int, int): %1$s", e.getMessage()); - } - return Bytes.DECODER_ERROR; - } - - /** - * Encode a String. - * - * @param encoder the encoder to use - * @param chars the string to encode - * @return the encoded Bytes - */ - public static Bytes encode(CharsetEncoder encoder, final String chars) { - return Bytes.encode(encoder, CharBuffer.wrap(chars.toCharArray())); - } - - /** - * Encode a CharBuffer. - * - * @param encoder the encoder to use - * @param chars the CharBuffer to encode - * @return the encoded Bytes - */ - public static Bytes encode(CharsetEncoder encoder, final CharBuffer chars) { - Bytes bytes = null; - try { - ByteBuffer buffer = encoder.reset().encode(chars); - byte[] b = new byte[buffer.limit()]; - buffer.get(b, 0, b.length); - bytes = new Bytes(b); - } catch (CharacterCodingException e) { - assert false; - System.err.println("Bytes.encode(CharsetEncoder,CharBuffer): " + e.getMessage()); - } - return bytes; - } - /** * Get the number of contained bytes. * @@ -132,8 +77,11 @@ public int getLength() { } /** - * Get the contained bytes. This is a direct reference to the wrapped array. Sadly - * there is no way for Java to confer immutability to the returned value. + * Get the backing array for the contained bytes. This is a direct reference to the + * wrapped array, which may not be completely filled; use Bytes.getLength() to determine + * actual length of data. Sadly there is no way for Java to confer immutability to the + * returned value, so don't mess with the contents unless you think you know what you + * are going. * * @return the contained bytes */ @@ -144,39 +92,58 @@ public byte[] bytes() { /** * Decode contents to a String. * - * @param decoder the decoder to use * @return a Unicode string */ - public String toString(CharsetDecoder decoder) { + public String asString() { try { - return decoder.reset().decode(ByteBuffer.wrap(this.bytes())).toString(); - } catch (Exception e) { + return Codec.decode(this.bytes(), this.getLength()); + } catch (CharacterCodingException e) { return this.toHexString(); } } - /** - * Decode contents to a String. - * - * @return a Unicode string - */ - @Override - public String toString() { - return this.toString(Base.newCharsetDecoder()); - } - /** * Lazy hash code evaluation. * * @return a hash based on the contents */ - @Override public int hashCode() { if (this.hash == 0) { this.hash = this.hash(); } return this.hash; } + + /** + * Like toString(), but the result is a hexadecimal representation of the contents. + * + * @return the hex string + */ + public String toHexString() { + char[] hex = new char[4 * this.getLength()]; + int i = 0; + for (int j = 0; j < this.getLength(); j++) { + byte b = this.data[j]; + if (b >= 32) { + hex[i++] = (char)(b & 0xFF); + } else { + hex[i++] = '\\'; + hex[i++] = 'x'; + hex[i++] = HEX[b >>> 4]; + hex[i++] = HEX[b & 0x0F]; + } + } + return new String(Arrays.copyOf(hex, i)); + } + + /** + * Override + * + * @return decoded string + */ + public String toString() { + return this.asString(); + } /** * Test for equality of contents. @@ -188,21 +155,6 @@ public int hashCode() { public boolean equals(final Object other) { return other instanceof Bytes o && Arrays.equals(o.data, this.data); } - - /** - * Like toString(), but the result is a hexadecimal representation of the contents. - * - * @return the hex string - */ - public String toHexString() { - char[] hex = new char[2 * this.getLength()]; - for (int j = 0; j < this.getLength(); j++) { - int k = this.data[j] & 0xFF; - hex[j * 2] = HEX[k >> 4]; - hex[j * 2 + 1] = HEX[k & 0x0F]; - } - return new String(hex); - } private int hash() { int h = 0; diff --git a/src/com/characterforming/ribose/base/SimpleTarget.java b/src/com/characterforming/ribose/base/SimpleTarget.java index a4ab137..6487357 100644 --- a/src/com/characterforming/ribose/base/SimpleTarget.java +++ b/src/com/characterforming/ribose/base/SimpleTarget.java @@ -42,7 +42,8 @@ public SimpleTarget() { } @Override // ITarget#getEffectors() - public IEffector[] getEffectors() throws TargetBindingException { + public IEffector[] getEffectors() + throws TargetBindingException { // This is just a proxy for Transductor.getEffectors() return new IEffector[] { }; }