From 65c2372bf892c3bcfb6ca8e00e5dc71910cd41c8 Mon Sep 17 00:00:00 2001 From: Alex Black Date: Thu, 7 May 2020 15:42:59 +1000 Subject: [PATCH] Proper image support in new pipeline/data API + Context/Metrics/Profiler API (placeholder) (#308) * API change: add Context to execution API with profiler() and metrics() methods + placeholder APIs Signed-off-by: Alex Black * Image JSON, PNG file support, class relocations, fixes Signed-off-by: Alex Black * BufferedImage support Signed-off-by: Alex Black * Add konduit-serving-javacv Signed-off-by: Alex Black * Add 2-step image conversion: X -> PNG -> Y when no direct X -> Y converter exists Signed-off-by: Alex Black * 2-step X -> SerializedNDArray -> Y conversion for NDArrays; fix image type conversion in konduit-serving-camera Signed-off-by: Alex Black * Update readme Signed-off-by: Alex Black --- dev_docs/README.md | 47 +++- .../konduit-serving-javacv/pom.xml | 68 +++++ .../serving/data/javacv/data/FrameImage.java | 28 ++ .../serving/data/javacv/data/MatImage.java | 28 ++ .../data/javacv/data/OpenCVMatImage.java | 34 +++ .../javacv/format/JavaCVImageConverters.java | 241 ++++++++++++++++++ .../javacv/format/JavaCVImageFactory.java | 67 +++++ .../serving/data/javacv/util/OpenCVUtil.java | 47 ++++ ...serving.pipeline.api.format.ImageConverter | 30 +++ ...t.serving.pipeline.api.format.ImageFactory | 20 ++ .../serving/data/javacv/TestConversion.java | 166 ++++++++++++ .../src/test/resources/data/5_32x32.png | Bin 0 -> 1095 bytes .../serving/data/nd4j/data/ND4JNDArray.java | 2 +- konduit-serving-data/pom.xml | 1 + .../konduit-serving-camera/pom.xml | 8 +- .../step/capture/FrameCaptureStepRunner.java | 5 +- .../camera/step/show/ShowImageStepRunner.java | 7 +- .../ai/konduit/serving/camera/ManualTest.java | 2 +- .../deeplearning4j/DL4JConfiguration.java | 46 ---- .../deeplearning4j/DL4JConfiguration.java | 48 ++++ .../serde/DL4JJsonSubTypesMapping.java | 6 +- .../step/DL4JModelPipelineStep.java | 4 +- .../step/DL4JPipelineStepRunner.java | 5 +- .../step/DL4JPipelineStepRunnerFactory.java | 2 +- ...ing.pipeline.api.serde.JsonSubTypesMapping | 2 +- ...ipeline.api.step.PipelineStepRunnerFactory | 2 +- .../deeplearning4j/TestDL4JModelStep.java | 20 +- .../serving/pipeline/api/context/Context.java | 35 +++ .../serving/pipeline/api/context/Metrics.java | 33 +++ .../pipeline/api/context/Profiler.java | 76 ++++++ .../serving/pipeline/api/data/Data.java | 14 +- .../serving/pipeline/api/data/Image.java | 20 +- .../serving/pipeline/api/data/ValueType.java | 30 +++ .../exception/DataConversionException.java | 36 +++ .../api/exception/DataLoadingException.java | 37 +++ .../pipeline/api/format/ImageConverter.java | 7 +- .../pipeline/api/format/NDArrayConverter.java | 2 +- .../api/pipeline/PipelineExecutor.java | 9 +- .../pipeline/api/step/PipelineStepRunner.java | 7 +- .../serving/pipeline/impl/data/JData.java | 1 + .../serving/pipeline/impl/data/Value.java | 2 + .../serving/pipeline/impl/data/ValueType.java | 28 -- .../pipeline/impl/data/image/BImage.java | 27 ++ .../pipeline/impl/data/image/BaseImage.java | 79 ++++++ .../serving/pipeline/impl/data/image/Png.java | 74 ++++++ .../JavaImage.java => image/PngImage.java} | 8 +- .../data/ndarray}/BaseNDArray.java | 7 +- .../ndarray}/SerializedNDArray.java | 2 +- .../impl/data/wrappers/BaseValue.java | 2 +- .../impl/data/wrappers/BooleanValue.java | 2 +- .../impl/data/wrappers/BytesValue.java | 2 +- .../impl/data/wrappers/DataValue.java | 2 +- .../impl/data/wrappers/DoubleValue.java | 2 +- .../impl/data/wrappers/ImageValue.java | 2 +- .../pipeline/impl/data/wrappers/IntValue.java | 2 +- .../impl/data/wrappers/NDArrayValue.java | 4 +- .../impl/data/wrappers/StringValue.java | 2 +- .../impl/format/JavaImageConverters.java | 132 ++++++++++ .../impl/format/JavaImageFactory.java | 81 ++++++ .../impl/format/JavaNDArrayConverters.java | 3 +- .../java => format}/JavaNDArrayFactory.java | 4 +- .../{data/java => format}/JavaNDArrays.java | 8 +- .../pipeline/SequencePipelineExecutor.java | 5 +- .../impl/serde/DataJsonDeserializer.java | 13 +- .../impl/serde/DataJsonSerializer.java | 15 +- .../logging/LoggingPipelineStepRunner.java | 3 +- .../pipeline/registry/AbstractRegistry.java | 6 + .../registry/ImageConverteRegistry.java | 61 ----- .../registry/ImageConverterRegistry.java | 144 +++++++++++ .../registry/ImageFactoryRegistry.java | 4 + .../registry/NDArrayConverterRegistry.java | 52 +++- .../registry/NDArrayFactoryRegistry.java | 5 + .../serving/pipeline/util/FileUtils.java | 41 +++ ...serving.pipeline.api.format.ImageConverter | 21 ++ ...t.serving.pipeline.api.format.ImageFactory | 20 ++ ...serving.pipeline.api.format.NDArrayFactory | 2 +- .../pipeline/impl/data/DataJsonTest.java | 7 +- .../pipeline/impl/data/ImageTests.java | 216 ++++++++++++++++ .../pipeline/impl/data/JavaFormatsTest.java | 4 +- .../pipeline/impl/data/NDArrayTests.java | 157 ++++++++++++ .../pipeline/impl/step/TestPipelineSteps.java | 4 +- .../impl/util/CallbackPipelineStep.java | 3 +- .../src/test/resources/data/5_32x32.png | Bin 0 -> 1095 bytes 83 files changed, 2271 insertions(+), 230 deletions(-) create mode 100644 konduit-serving-data/konduit-serving-javacv/pom.xml create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/FrameImage.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/MatImage.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/OpenCVMatImage.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageConverters.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageFactory.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/util/OpenCVUtil.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter create mode 100644 konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory create mode 100644 konduit-serving-data/konduit-serving-javacv/src/test/java/ai/konduit/serving/data/javacv/TestConversion.java create mode 100644 konduit-serving-data/konduit-serving-javacv/src/test/resources/data/5_32x32.png delete mode 100644 konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/DL4JConfiguration.java create mode 100644 konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/DL4JConfiguration.java rename konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/{ => models}/deeplearning4j/serde/DL4JJsonSubTypesMapping.java (88%) rename konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/{ => models}/deeplearning4j/step/DL4JModelPipelineStep.java (93%) rename konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/{ => models}/deeplearning4j/step/DL4JPipelineStepRunner.java (97%) rename konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/{ => models}/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java (96%) create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Context.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Metrics.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Profiler.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/ValueType.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataConversionException.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataLoadingException.java delete mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ValueType.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BImage.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BaseImage.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/Png.java rename konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/{java/JavaImage.java => image/PngImage.java} (83%) rename konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/{api/data => impl/data/ndarray}/BaseNDArray.java (92%) rename konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/{format => data/ndarray}/SerializedNDArray.java (97%) create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageConverters.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageFactory.java rename konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/{data/java => format}/JavaNDArrayFactory.java (95%) rename konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/{data/java => format}/JavaNDArrays.java (86%) delete mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverteRegistry.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverterRegistry.java create mode 100644 konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/util/FileUtils.java create mode 100644 konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter create mode 100644 konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory create mode 100644 konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/ImageTests.java create mode 100644 konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/NDArrayTests.java create mode 100644 konduit-serving-pipeline/src/test/resources/data/5_32x32.png diff --git a/dev_docs/README.md b/dev_docs/README.md index 1ed604c76..dbbeaa6ab 100644 --- a/dev_docs/README.md +++ b/dev_docs/README.md @@ -29,7 +29,8 @@ The list below briefly describes those implemented so far (as of 05/05/2020)- th * konduit-serving-models: Parent module for each of the model types * konduit-serving-deeplearning4j: Deeplearning4j models. * konduit-serving-data: Parent module for data and datatypes - * konduit-serving-nd4j: Mainly NDArray integration/functionality for ND4J + * konduit-serving-nd4j: Mainly NDArray integration/functionality for ND4J + * konduit-serving-javacv: Image conversion functionality for JavaCV * konduit-serving-io: Parent module for I/O functionality - sensors, cameras, etc - and maybe later things like HDFS, S3, etc * konduit-serving-camera: Steps related to capturing data from device-connected cameras (WIP) @@ -248,6 +249,20 @@ ByteBuffer bb = sArr.getBuffer(); All modules that define an NDArray type should implement conversion to SerializedNDArray, so we can do JSON and Protobuf serialization from any format - in a standardized form. +As of 07/05/2020 supported formats for images include: +* Png +* BufferedImage +* JavaCV Mat (konduit-serving-javacv) +* JavaCV Frame (konduit-serving-javacv) +* OpenCV Mat (konduit-serving-javacv) +More formats will be added in the future. + +As of 07/05/2020 supported NDArray formats include: +* SerializedNDArray (Konduit Serving serialization/interchange format) +* float[], float[][], float[][][], float[][][][], float[][][][][] +* INDArray (konduit-serving-nd4j) +The full set of Java primitive array types (1d to 5d int[], double[], byte[] etc is planned to be added) + ## JSON Serialization / Deserialization @@ -314,6 +329,13 @@ As for the actual JSON format - this is basically as follows (taken from DataJso "myKey" : { "@BytesBase64" : "AAEC" } +} + ----- IMAGE ----- +{ + "myKey" : { + "@ImageFormat" : "PNG", + "@ImageData" : "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAPcSURBVFhH7VfZK3VRFF/XNRSZyTyHyAOSUqZHpRCRPCjlzZM/QH3/AU+G4oHyJK9KmQohJSKZZxkjmcf1rbWce13Hce5xP3Vfvl/92ufss4ffXnuttfcxAQASnYKkpCRwUZ6dhv8CnC7gn5zQZDIB4nv39PR08PHxAX9/fzg+PoalpSUoKyt7dzQXF5ienoaZmRk4Pz+X9gz+ZkgAD8ATeXt7ywReXl7CqKgoeHl5gcfHR5ksODgYQkNDYXFxEQYHB6G5uRliY2NljJaWFmhra4O1tTV5Z9gVwCt0dXWFyspKmYgHS0tLg7i4OAgLC4OEhASl5QeOjo5gYWEBrq+vIScnR6yxvr4Ora2tMD8/Dzc3N0pLAwI8PDwgIiICNjc3lZrPeHt7E7KFGFx2dnZCf3+/iL+/vxdBt7e3cHBwIG1swQIYLECTNKCUr6+vtAMfsH2nVeHw8LDyhtjQ0PBlnO9IAtBMD3+ImqDxpFxZWZFV9PX1wcPDAyQnJ0vJe52RkQE9PT3Q0dEhfjE3Nwerq6vSzx4CAwOl/KJMTV9fX0xMTMSAgACsqqrCra0t3N7extraWmsbs9mM0dHR0ta2rx7ZAobywNXVlfjBxcUFnJ6eioP5+flBamqq0gKAtgX29vak7U9gOBGxszGWl5clxHh76urqrGZ0FLo+oAX2BV5lSEgIZGdnW8OKTC8Wsgg1AsM+oCZHR2FhIVKMIzkjjoyMYFdXF1I2RApdpBDU7Kcm+wCV2h/t0dPTE0tLSyX0LOju7sbc3FwRodVHzX8SYCGlZ5yYmFAkIFIWRPINaw7R468IYLKI4uJi3NnZERGHh4fY3t6u2daWvybAQjrxRABjf38fKUw121loOA/ogT25uroaGhsbgRKR1N3d3cHu7q6ErD04dB/ggyYyMlJOu7y8PMjKyoKgoCCIiYkBNzc3SVRDQ0NQXl6u9NAGWUDKL6bRI+93QUEBNjU14eTkJD49PSlGR6QjGzc2NpBOQ0OHEm+BIQtQnpejmW89vOKKigrIzMyUegYnJ07Ds7OzMDAwAJQX4OzsTL7pwa4FOJTItJiSkoJFRUVIg+Pz87OyXkS6CSFlQhwdHcWamhrNMfRoNwro1oNjY2N4eXmpTPkBFjI1NYX5+fmafY1QUwBPyolkfHwcyZuV6T6DLpxYX1//qZ8j1BRQUlIiq7YF7Sf29vZKvueJw8PDJRWr+/6Umk5Ig4tzcElmlpDjsKLsJifdycmJIQczAp7HoTzwW2ABTv0zEgtziHH2UoM/quv534D/E9T4rp6SFLi7uytv7yCXkusbb218fDz8Bd+qYeQWjEzlAAAAAElFTkSuQmCC" + } } ----- DOUBLE ----- { @@ -330,8 +352,8 @@ As for the actual JSON format - this is basically as follows (taken from DataJso ----- DATA ----- { "myKey" : { - "myInnerKey" : "myInnerValue" - } + "myInnerKey" : "myInnerValue" + } } ``` @@ -342,6 +364,8 @@ Bytes, is a special case - it's an object with a special protected key: `@BytesB will encode `byte[]` objects as base64 format for space efficiency; later we will allow "array style" byte[] encoding (i.e., `"@BytesArray" : [1,2,3]`). Users are not allowed to add anything to a Data instance with these protected keys. +Image data is stored by default in PNG format, base64 encoded (this will be configurable eventually). + NDArray is a special case also: it in a JSON object with type/shape/data keys. Currently data is base64 encoded, but we may allow a "1d buffer array" format in the future also. @@ -353,7 +377,6 @@ The full set of protected keys can be found on the Data interface. They include: * @NDArrayShape * @NDArrayType * @NDArrayDataBase64 -* @NDArrayDataBase64 * @Metadata In practice, Data JSON serialization/deserialization is implemneted in the DataJsonSerializer and DataJsonDeserializer classes. @@ -390,6 +413,13 @@ See for example: konduit-serving-deeplearning4j - For NDArray: the main (strictly required) one is conversion to/from SerializedNDArray - this is necessary for JSON and protobuf serialization/deserialization, and will be used as the intermediate format for conversion between arbitrary types that don't have direct (1 step) conversion enabled (i.e., X -> SerializedNDArray -> Y for any X and Y). + - For Image: the main (strictly required) one is conversion to/from Png (i.e., `ai.konduit.serving.pipeline.impl.data.image.Png`) + as this is used as the default format for both JSON and Protobuf serialization - and also used as the intermediate + format for conversion between arbitrary image formats that don't have direct (1 step) conversion available (i.e., X + -> Png -> Y for any X and Y). + The reason PNG was chosen: PNG is a compressed lossless image format, unlike some alternatives such as jpeg. It does + have a size overhead for natural images vs. jpeg, but in practice for deep learning we typically don't have very large + input/output images so this is a secondary concern. - Add an NDArrayFactory / ImageFactory (used within NDArray.create(Object) / Image.create(Object)) - Add a `resources/META-INF/services/ai.konduit.serving.pipeline.api.format.NDArrayConverter` (or `.ImageConverter`) file listing the fully-qualified class name of all of the new NDArrayConverter/ImageConverter implementations you added @@ -399,6 +429,15 @@ See for example: konduit-serving-deeplearning4j See for example: konduit-serving-nd4j + +A note on naming packages for new modules: The packages (that directly contain classe) should be unique, and are based on +the module name. +For example, the kondit-serving-nd4j module (under konduit-serving-data) has classes in `ai.konduit.serving.data.nd4j`. +Similarly, konduit-serving-deeplearning4j (under konduit-serving-models) has classes in `ai.konduit.serving.models.deeplearning4j`. +It's fine to have sub-packages (i.e., any X in `ai.konduit.serving.data.nd4j.X` is fine) but we cannot have classes +in the same package - i.e., modules X and Y can't both define classes directly in the same namespace. +The reason is to avoid split packages issues for OSGi and Java 9 Modules. We will likely use OSGi extensively in the future. + diff --git a/konduit-serving-data/konduit-serving-javacv/pom.xml b/konduit-serving-data/konduit-serving-javacv/pom.xml new file mode 100644 index 000000000..af4146e99 --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/pom.xml @@ -0,0 +1,68 @@ + + + + + + konduit-serving-data + ai.konduit.serving + 0.1.0-SNAPSHOT + + 4.0.0 + + konduit-serving-javacv + + + + ai.konduit.serving + konduit-serving-pipeline + ${project.version} + + + + org.projectlombok + lombok + + + + org.bytedeco + javacv-platform + ${javacpp.version} + + + + + + junit + junit + test + + + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + + + \ No newline at end of file diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/FrameImage.java b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/FrameImage.java new file mode 100644 index 000000000..1bc18968e --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/FrameImage.java @@ -0,0 +1,28 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv.data; + +import ai.konduit.serving.pipeline.impl.data.image.BaseImage; +import org.bytedeco.javacv.Frame; + +public class FrameImage extends BaseImage { + public FrameImage(Frame image) { + super(image); + } +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/MatImage.java b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/MatImage.java new file mode 100644 index 000000000..942a96a10 --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/MatImage.java @@ -0,0 +1,28 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv.data; + +import ai.konduit.serving.pipeline.impl.data.image.BaseImage; +import org.bytedeco.opencv.opencv_core.Mat; + +public class MatImage extends BaseImage { + public MatImage(Mat image) { + super(image); + } +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/OpenCVMatImage.java b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/OpenCVMatImage.java new file mode 100644 index 000000000..f019a1e6f --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/data/OpenCVMatImage.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv.data; + +import ai.konduit.serving.data.javacv.util.OpenCVUtil; +import ai.konduit.serving.pipeline.impl.data.image.BaseImage; +import org.opencv.core.Mat; + +public class OpenCVMatImage extends BaseImage { + + static { + OpenCVUtil.ensureOpenCVLoaded(); + } + + public OpenCVMatImage(Mat image) { + super(image); + } +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageConverters.java b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageConverters.java new file mode 100644 index 000000000..4a3a02e85 --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageConverters.java @@ -0,0 +1,241 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv.format; + +import ai.konduit.serving.data.javacv.util.OpenCVUtil; +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.exception.DataConversionException; +import ai.konduit.serving.pipeline.api.format.ImageConverter; +import ai.konduit.serving.pipeline.api.format.ImageFormat; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.format.JavaImageConverters; +import ai.konduit.serving.pipeline.util.FileUtils; +import lombok.AllArgsConstructor; +import org.apache.commons.io.IOUtils; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.OpenCVFrameConverter; +import org.bytedeco.opencv.opencv_core.Mat; +import org.nd4j.common.base.Preconditions; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +public class JavaCVImageConverters { + + private JavaCVImageConverters(){ } + + static { + OpenCVUtil.ensureOpenCVLoaded(); + } + + public static class FrameToMatConverter extends JavaImageConverters.BaseConverter { + protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + + public FrameToMatConverter() { + super(Frame.class, Mat.class); + } + + @Override + protected T doConversion(Image from, Class to) { + Frame f = (Frame) from.get(); + Mat m = converter.convert(f); + return (T)m; + } + } + + public static class MatToFrameConverter extends JavaImageConverters.BaseConverter { + protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + + public MatToFrameConverter() { + super(Mat.class, Frame.class); + } + + @Override + protected T doConversion(Image from, Class to) { + Mat m = (Mat) from.get(); + Frame f = converter.convert(m); + return (T)f; + } + } + + public static class FrameToOpenCVMatConverter extends JavaImageConverters.BaseConverter { + protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + + public FrameToOpenCVMatConverter() { + super(Frame.class, org.opencv.core.Mat.class); + } + + @Override + protected T doConversion(Image from, Class to) { + Frame f = (Frame) from.get(); + Mat m = converter.convert(f); + org.opencv.core.Mat m2 = new org.opencv.core.Mat(m.address()); + return (T)m2; + } + } + + public static class OpenCVMatToFrameConverter extends JavaImageConverters.BaseConverter { + protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + + public OpenCVMatToFrameConverter() { + super(org.opencv.core.Mat.class, Frame.class); + } + + @Override + protected T doConversion(Image from, Class to) { + org.opencv.core.Mat m = (org.opencv.core.Mat) from.get(); + Frame f = converter.convert(m); + return (T)f; + } + } + + public static class MatToOpenCVMatConverter extends JavaImageConverters.BaseConverter { + public MatToOpenCVMatConverter() { + super(Mat.class, org.opencv.core.Mat.class); + } + + @Override + protected T doConversion(Image from, Class to) { + Mat m = (Mat) from.get(); + org.opencv.core.Mat m2 = new org.opencv.core.Mat(m.address()); + return (T) m2; + } + } + + public static class OpenCVMatToMatConverter extends JavaImageConverters.BaseConverter { + + public OpenCVMatToMatConverter() { + super(org.opencv.core.Mat.class, Mat.class); + } + + @Override + protected T doConversion(Image from, Class to) { + org.opencv.core.Mat m = (org.opencv.core.Mat) from.get(); + Mat m2 = new Mat((Pointer)null){{address = m.getNativeObjAddr();}}; + return (T)m2.clone(); + } + } + + public static class MatToPng extends JavaImageConverters.BaseConverter { + public MatToPng() { + super(Mat.class, Png.class); + } + + @Override + protected T doConversion(Image from, Class to) { + //TODO It may be possible to do this without the temp file + Mat m = (Mat) from.get(); + File tempDir = FileUtils.getTempFileDir("konduit-serving-javacv"); + File f = new File(tempDir, UUID.randomUUID().toString() + ".png"); + String path = f.getAbsolutePath(); + + org.bytedeco.opencv.global.opencv_imgcodecs.imwrite(path, m); + try { + byte[] bytes = org.apache.commons.io.FileUtils.readFileToByteArray(f); + f.delete(); + return (T) new Png(bytes); + } catch (IOException e){ + throw new DataConversionException("Error connverting Mat to Png", e); + } + } + } + + public static class PngToMat extends JavaImageConverters.BaseConverter { + public PngToMat() { + super(Png.class, Mat.class); + } + + @Override + protected T doConversion(Image from, Class to) { + //TODO is there a way to do this without the temp file? + Png p = (Png) from.get(); + byte[] bytes = p.getBytes(); + File tempDir = FileUtils.getTempFileDir("konduit-serving-javacv"); + File f = new File(tempDir, UUID.randomUUID().toString() + ".png"); + Mat mat; + try { + org.apache.commons.io.FileUtils.writeByteArrayToFile(f, bytes); + mat = org.bytedeco.opencv.global.opencv_imgcodecs.imread(f.getAbsolutePath()); + } catch (IOException e){ + throw new DataConversionException("Error writing to temporary file for Png->Mat conversion", e); + } + f.delete(); + return (T) mat; + } + } + + public static class FrameToPng extends JavaImageConverters.BaseConverter { + public FrameToPng() { + super(Frame.class, Png.class); + } + + @Override + protected T doConversion(Image from, Class to) { + //Frame -> Mat -> Png. Is there a more efficient way? + Frame f = (Frame) from.get(); + Mat m = Image.create(f).getAs(Mat.class); + return (T) Image.create(m).getAs(Png.class); + } + } + + public static class PngToFrame extends JavaImageConverters.BaseConverter { + public PngToFrame() { + super(Png.class, Frame.class); + } + + @Override + protected T doConversion(Image from, Class to) { + //Png -> Mat -> Frame. Is there a more efficient way? + Png p = (Png)from.get(); + Mat m = Image.create(p).getAs(Mat.class); + return (T) Image.create(m).getAs(Frame.class); + } + } + + public static class OpenCVMatToPng extends JavaImageConverters.BaseConverter { + public OpenCVMatToPng() { + super(org.opencv.core.Mat.class, Png.class); + } + + @Override + protected T doConversion(Image from, Class to) { + //org.opencv.core.Mat -> Mat -> Png. Is there a more efficient way? + org.opencv.core.Mat m = (org.opencv.core.Mat) from.get(); + Mat m2 = Image.create(m).getAs(Mat.class); + return (T) Image.create(m2).getAs(Png.class); + } + } + + public static class PngToOpenCVMat extends JavaImageConverters.BaseConverter { + public PngToOpenCVMat() { + super(Png.class, org.opencv.core.Mat.class); + } + + @Override + protected T doConversion(Image from, Class to) { + //Png -> Mat -> org.opencv.core.Mat. Is there a more efficient way? + Png p = (Png)from.get(); + Mat m = Image.create(p).getAs(Mat.class); + return (T) Image.create(m).getAs(org.opencv.core.Mat.class); + } + } +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageFactory.java b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageFactory.java new file mode 100644 index 000000000..5dc68a167 --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/format/JavaCVImageFactory.java @@ -0,0 +1,67 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv.format; + +import ai.konduit.serving.data.javacv.data.FrameImage; +import ai.konduit.serving.data.javacv.data.OpenCVMatImage; +import ai.konduit.serving.data.javacv.data.MatImage; +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.format.ImageFactory; +import org.bytedeco.javacv.Frame; +import org.nd4j.common.base.Preconditions; +import org.opencv.core.Mat; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class JavaCVImageFactory implements ImageFactory { + + private static Set> SUPPORTED_TYPES = new HashSet<>(); + static { + SUPPORTED_TYPES.add(Frame.class); + SUPPORTED_TYPES.add(Mat.class); + SUPPORTED_TYPES.add(org.bytedeco.opencv.opencv_core.Mat.class); + } + + @Override + public Set> supportedTypes() { + return Collections.unmodifiableSet(SUPPORTED_TYPES); + } + + @Override + public boolean canCreateFrom(Object o) { + return SUPPORTED_TYPES.contains(o.getClass()); + } + + @Override + public Image create(Object o) { + Preconditions.checkState(canCreateFrom(o), "Unable to create Image from object of type %s", o.getClass()); + + if(o instanceof Frame){ + return new FrameImage((Frame) o); + } else if(o instanceof Mat){ + return new OpenCVMatImage((Mat)o); + } else if(o instanceof org.bytedeco.opencv.opencv_core.Mat){ + return new MatImage((org.bytedeco.opencv.opencv_core.Mat) o); + } else { + throw new IllegalStateException("Unable to create image from format " + o.getClass()); + } + } +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/util/OpenCVUtil.java b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/util/OpenCVUtil.java new file mode 100644 index 000000000..64936fb31 --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/java/ai/konduit/serving/data/javacv/util/OpenCVUtil.java @@ -0,0 +1,47 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv.util; + +import org.bytedeco.javacpp.Loader; +import org.bytedeco.opencv.opencv_java; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class OpenCVUtil { + + private OpenCVUtil(){ } + + + private static final AtomicBoolean opencvLoaded = new AtomicBoolean(); + + /** + * + */ + public static synchronized void ensureOpenCVLoaded(){ + if(opencvLoaded.get()) + return; + + /* + Call Loader.load(opencv_java.class) before using the API in the org.opencv namespace. + */ + Loader.load(opencv_java.class); + + } + +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter b/konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter new file mode 100644 index 000000000..89da4017f --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter @@ -0,0 +1,30 @@ +# +# /* ****************************************************************************** +# * Copyright (c) 2020 Konduit K.K. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$FrameToMatConverter +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$MatToFrameConverter +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$FrameToOpenCVMatConverter +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$OpenCVMatToFrameConverter +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$MatToOpenCVMatConverter +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$OpenCVMatToMatConverter +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$MatToPng +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$PngToMat +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$FrameToPng +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$PngToFrame +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$OpenCVMatToPng +ai.konduit.serving.data.javacv.format.JavaCVImageConverters$PngToOpenCVMat \ No newline at end of file diff --git a/konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory b/konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory new file mode 100644 index 000000000..e2ff3ab8e --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory @@ -0,0 +1,20 @@ +# +# /* ****************************************************************************** +# * Copyright (c) 2020 Konduit K.K. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + + +ai.konduit.serving.data.javacv.format.JavaCVImageFactory \ No newline at end of file diff --git a/konduit-serving-data/konduit-serving-javacv/src/test/java/ai/konduit/serving/data/javacv/TestConversion.java b/konduit-serving-data/konduit-serving-javacv/src/test/java/ai/konduit/serving/data/javacv/TestConversion.java new file mode 100644 index 000000000..b1761af81 --- /dev/null +++ b/konduit-serving-data/konduit-serving-javacv/src/test/java/ai/konduit/serving/data/javacv/TestConversion.java @@ -0,0 +1,166 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.data.javacv; + +import ai.konduit.serving.pipeline.api.data.Data; +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.OpenCVFrameConverter; +import org.bytedeco.opencv.opencv_core.Mat; +import org.junit.Test; +import org.nd4j.common.resources.Resources; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestConversion { + + @Test + public void testConversion(){ + /* + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$FrameToMatConverter + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$MatToFrameConverter + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$FrameToOpenCVMatConverter + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$OpenCVMatToFrameConverter + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$MatToOpenCVMatConverter + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$OpenCVMatToMatConverter + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$MatToPng + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$PngToMat + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$FrameToPng + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$PngToFrame + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$OpenCVMatToPng + ai.konduit.serving.data.javacv.format.JavaCVImageConverters$PngToOpenCVMat + */ + + File f = Resources.asFile("data/5_32x32.png"); + + OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + Mat mat = org.bytedeco.opencv.global.opencv_imgcodecs.imread(f.getAbsolutePath()); + Frame frame = converter.convert(mat); + org.opencv.core.Mat opencvMat = new org.opencv.core.Mat(mat.address()); + + Image iMat = Image.create(mat); + Image iFrame = Image.create(frame); + Image iCVMat = Image.create(opencvMat); + + //To Mat: + Mat m1 = iMat.getAs(Mat.class); + Mat m2 = iFrame.getAs(Mat.class); + Mat m3 = iCVMat.getAs(Mat.class); + assertTrue(equalMats(m1, m2)); + assertTrue(equalMats(m1, m3)); + + //To Frame: + Frame f1 = iMat.getAs(Frame.class); + Frame f2 = iFrame.getAs(Frame.class); + Frame f3 = iCVMat.getAs(Frame.class); + assertTrue(equalFrames(f1, f2)); + assertTrue(equalFrames(f1, f3)); + + //To OpenCVMat: + org.opencv.core.Mat om1 = iMat.getAs(org.opencv.core.Mat.class); + org.opencv.core.Mat om2 = iFrame.getAs(org.opencv.core.Mat.class); + org.opencv.core.Mat om3 = iCVMat.getAs(org.opencv.core.Mat.class); + assertTrue(equalOpenCvMats(om1, om2)); + assertTrue(equalOpenCvMats(om1, om3)); + + + + //To PNG: + Png p1 = iMat.getAs(Png.class); + Png p2 = iFrame.getAs(Png.class); + Png p3 = iCVMat.getAs(Png.class); + assertTrue(equalPngs(p1, p2)); + assertTrue(equalPngs(p1, p3)); + + //From PNG: + Image png = Image.create(p1); + Mat pm = png.getAs(Mat.class); + Frame pf = png.getAs(Frame.class); + org.opencv.core.Mat pcvm = png.getAs(org.opencv.core.Mat.class); + assertTrue(equalMats(mat, pm)); + assertTrue(equalFrames(frame, pf)); + assertTrue(equalOpenCvMats(opencvMat, pcvm)); + + + + //Test Data + for(Image i : new Image[]{iMat, iFrame, iCVMat}){ + Data d = Data.singleton("myImage", i); + String json = d.toJson(); + Data dJson = Data.fromJson(json); + + assertEquals(d, dJson); + } + + + } + + protected static boolean equalMats(Mat m1, Mat m2){ + Png p1 = Image.create(m1).getAs(Png.class); + Png p2 = Image.create(m2).getAs(Png.class); + return equalPngs(p1, p2); + } + + protected static boolean equalFrames(Frame f1, Frame f2){ + Png p1 = Image.create(f1).getAs(Png.class); + Png p2 = Image.create(f2).getAs(Png.class); + return equalPngs(p1, p2); + } + + protected static boolean equalOpenCvMats(org.opencv.core.Mat m1, org.opencv.core.Mat m2){ + Png p1 = Image.create(m1).getAs(Png.class); + Png p2 = Image.create(m2).getAs(Png.class); + return equalPngs(p1, p2); + } + + protected static boolean equalPngs(Png png1, Png png2){ + try { + BufferedImage bi1 = ImageIO.read(new ByteArrayInputStream(png1.getBytes())); + BufferedImage bi2 = ImageIO.read(new ByteArrayInputStream(png2.getBytes())); + return bufferedImagesEqual(bi1, bi2); + } catch (Throwable t){ + throw new RuntimeException(t); + } + } + + protected static boolean bufferedImagesEqual(BufferedImage img1, BufferedImage img2) { + if (img1.getHeight() != img2.getHeight() || img1.getWidth() != img2.getWidth()) { + return false; + } + int w = img1.getWidth(); + int h = img1.getHeight(); + + for (int i = 0; i < h; i++) { + for (int j = 0; j < w; j++) { + if (img1.getRGB(i,j) != img2.getRGB(i,j)) { + return false; + } + } + } + return true; + } + +} diff --git a/konduit-serving-data/konduit-serving-javacv/src/test/resources/data/5_32x32.png b/konduit-serving-data/konduit-serving-javacv/src/test/resources/data/5_32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..a21967b8acfbf15dd6a7980e08f22f7fd9681827 GIT binary patch literal 1095 zcmV-N1i1T&P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf1KdeOK~z{r?N`|= zbx{;w*EJNG$vlV11CmmvN2L&vJSgSOlRrRx{{c^i;(_ucuS%H;AtfT2XBi`z$MvnX zoO@lzaqe+Hbzi<;{o3d3vxnbad#$ziF#!M)or0vKBydrloreDcoG##>oX})40pff2 z^z?M_@$mtFe}6bWK8B^GB`7N^dv`P!7nq)&hMAceI6ps^_6+`JNB|GO6M1`k%LHCt zUc#uTD0q2!fyc*3nJhRs7(zosVR3O01_uYBxw#o)V`F8EmX;Q1YionGwKaKX*cHGl zbai!w%F0TahzCncOM|$$I0y?1gM@?xrR4{Xj*ehqVF9kMuOTNVN35~Fz7DOetuQ}7 z4>vbAN=pL*JUl!gA|e7dH#e1<&v<)#6S9RETy}PLLVth1h`+zT7eTkTw{UQ9AZ={H z0vIenlQf`|udlDP10S$`r&Zzb@UZfR*wE1MWt{Iw0?!r_4UES z!viEICrinDosp3NJv}|p(a|A(F*i2{tE;Qh&mIB;1Eu_;Oif>3UrbC)#DIVRtg5QQ zt*tHG-rmOA+FENHv)PQ%(b3Y@wyz%90*3Oy)zy{w!NtV|oSvSFgZcUSL27EM(gJ|C z1@`v#r0q{&I1#b4F_xE?CB%4p>gwvOW^@#~KA;8m1zu%HNC;$SXIsOHOfMF)2sHx3 zKAL5?~>vXPlUrP$VEy7DVz5YYx%luLxkWh~na6+}YU?5r>C|*xuf*+18f7 zCcr{E@k9V)e}5k-Of}nCXgCk(fjzmpx*8fA8-Ws$)ZN|P!S3#^gseR{bRXb@CNeTo zPP@FkJjlw*f}o%vh>3{-H#av)R6|2UP+nfH^w0)bK*}$s(IfA7K|ujFH8tVnLqmrWZ-F5TP@b2gnQuM3;);q2$jr=?dIl#aPQ$aavoJ6)0F)Qt z?Ceaw>A|iAIF!h=n39r$MMXtOgX8n_v$9@9x(Oj=LL40(#hRKL%`x<7Hwx(1gJWZ3 zczJoLO!)y8GBq`Y`T6;peT`6408h$^WPE%a$(xnwK7%ftSYKcNsh=a%6i`x9BCECq z?I-s3_Da6ziNnLgB^9gt{*;?SAO$;pW!2Qvbm$Dus%TVMfB z{WB5^bBJ;FQiDIL>-pb+{}a^suAY( { diff --git a/konduit-serving-data/pom.xml b/konduit-serving-data/pom.xml index e16700aa4..4e961ca8f 100644 --- a/konduit-serving-data/pom.xml +++ b/konduit-serving-data/pom.xml @@ -32,6 +32,7 @@ pom konduit-serving-nd4j + konduit-serving-javacv diff --git a/konduit-serving-io/konduit-serving-camera/pom.xml b/konduit-serving-io/konduit-serving-camera/pom.xml index 33476e77e..0fcef7128 100644 --- a/konduit-serving-io/konduit-serving-camera/pom.xml +++ b/konduit-serving-io/konduit-serving-camera/pom.xml @@ -13,14 +13,14 @@ - org.bytedeco - javacv-platform - ${javacpp.version} + ai.konduit.serving + konduit-serving-pipeline + ${project.version} ai.konduit.serving - konduit-serving-pipeline + konduit-serving-javacv ${project.version} diff --git a/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/capture/FrameCaptureStepRunner.java b/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/capture/FrameCaptureStepRunner.java index 0548bb00b..581dacb75 100644 --- a/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/capture/FrameCaptureStepRunner.java +++ b/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/capture/FrameCaptureStepRunner.java @@ -17,6 +17,7 @@ */ package ai.konduit.serving.camera.step.capture; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.step.PipelineStep; @@ -55,13 +56,13 @@ public PipelineStep getPipelineStep() { } @Override - public synchronized Data exec(Data data) { + public synchronized Data exec(Context ctx, Data data) { if(!initialized) init(); try { Frame frame = grabber.grab(); - Image i = Image.create(frame); //TODO Image format is yet to be determined! + Image i = Image.create(frame); return Data.singleton(step.getOutputKey(), i); } catch (Throwable t){ throw new RuntimeException("Error getting frame", t); diff --git a/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/show/ShowImageStepRunner.java b/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/show/ShowImageStepRunner.java index 68ba1697b..217c7999e 100644 --- a/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/show/ShowImageStepRunner.java +++ b/konduit-serving-io/konduit-serving-camera/src/main/java/ai/konduit/serving/camera/step/show/ShowImageStepRunner.java @@ -18,11 +18,12 @@ package ai.konduit.serving.camera.step.show; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.step.PipelineStep; import ai.konduit.serving.pipeline.api.step.PipelineStepRunner; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.Frame; import org.nd4j.common.base.Preconditions; @@ -51,7 +52,7 @@ public PipelineStep getPipelineStep() { } @Override - public synchronized Data exec(Data data) { + public synchronized Data exec(Context ctx, Data data) { String name = step.getImageName(); if(name == null) name = tryInferName(data); @@ -60,7 +61,7 @@ public synchronized Data exec(Data data) { name, data.keys()); Image i = data.getImage(name); - Frame f = (Frame)i.get(); //TODO NO CAST + Frame f = i.getAs(Frame.class); if(!initialized) init(); diff --git a/konduit-serving-io/konduit-serving-camera/src/test/java/ai/konduit/serving/camera/ManualTest.java b/konduit-serving-io/konduit-serving-camera/src/test/java/ai/konduit/serving/camera/ManualTest.java index d10f8f461..8c23042e2 100644 --- a/konduit-serving-io/konduit-serving-camera/src/test/java/ai/konduit/serving/camera/ManualTest.java +++ b/konduit-serving-io/konduit-serving-camera/src/test/java/ai/konduit/serving/camera/ManualTest.java @@ -42,7 +42,7 @@ public void manualTest() throws Exception { for( int i=0; i<100; i++ ) { - exec.exec(in); + exec.exec(null, in); Thread.sleep(100); } diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/DL4JConfiguration.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/DL4JConfiguration.java deleted file mode 100644 index b8161cba2..000000000 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/DL4JConfiguration.java +++ /dev/null @@ -1,46 +0,0 @@ -/* ****************************************************************************** - * Copyright (c) 2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ -package ai.konduit.serving.deeplearning4j; - -import ai.konduit.serving.pipeline.api.Configuration; - -import java.util.Collections; -import java.util.Map; -import java.util.Set; - -public class DL4JConfiguration implements Configuration { - - - @Override - public Set keys() { - return Collections.emptySet(); - } - - @Override - public Map asMap() { - return Collections.emptyMap(); - } - - @Override - public Object get(String key) { - throw new IllegalStateException("No key \"" + key + "\" exists"); - } - - @Override - public Object getOrDefault(String key, Object defaultValue) { - return defaultValue; - } -} diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/DL4JConfiguration.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/DL4JConfiguration.java new file mode 100644 index 000000000..d7c19b601 --- /dev/null +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/DL4JConfiguration.java @@ -0,0 +1,48 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ +package ai.konduit.serving.models.deeplearning4j; + +import ai.konduit.serving.pipeline.api.Configuration; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +public class DL4JConfiguration implements Configuration { + + + @Override + public Set keys() { + return Collections.emptySet(); + } + + @Override + public Map asMap() { + return Collections.emptyMap(); + } + + @Override + public Object get(String key) { + throw new IllegalStateException("No key \"" + key + "\" exists"); + } + + @Override + public Object getOrDefault(String key, Object defaultValue) { + return defaultValue; + } +} diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/serde/DL4JJsonSubTypesMapping.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/serde/DL4JJsonSubTypesMapping.java similarity index 88% rename from konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/serde/DL4JJsonSubTypesMapping.java rename to konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/serde/DL4JJsonSubTypesMapping.java index 6317980ed..e935abcfc 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/serde/DL4JJsonSubTypesMapping.java +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/serde/DL4JJsonSubTypesMapping.java @@ -13,10 +13,10 @@ * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ -package ai.konduit.serving.deeplearning4j.serde; +package ai.konduit.serving.models.deeplearning4j.serde; -import ai.konduit.serving.deeplearning4j.DL4JConfiguration; -import ai.konduit.serving.deeplearning4j.step.DL4JModelPipelineStep; +import ai.konduit.serving.models.deeplearning4j.DL4JConfiguration; +import ai.konduit.serving.models.deeplearning4j.step.DL4JModelPipelineStep; import ai.konduit.serving.pipeline.api.Configuration; import ai.konduit.serving.pipeline.api.serde.JsonSubType; import ai.konduit.serving.pipeline.api.serde.JsonSubTypesMapping; diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JModelPipelineStep.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JModelPipelineStep.java similarity index 93% rename from konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JModelPipelineStep.java rename to konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JModelPipelineStep.java index 7eede741e..c3cb95f27 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JModelPipelineStep.java +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JModelPipelineStep.java @@ -13,9 +13,9 @@ * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ -package ai.konduit.serving.deeplearning4j.step; +package ai.konduit.serving.models.deeplearning4j.step; -import ai.konduit.serving.deeplearning4j.DL4JConfiguration; +import ai.konduit.serving.models.deeplearning4j.DL4JConfiguration; import ai.konduit.serving.pipeline.api.BaseModelPipelineStep; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JPipelineStepRunner.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JPipelineStepRunner.java similarity index 97% rename from konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JPipelineStepRunner.java rename to konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JPipelineStepRunner.java index e5eb68bda..60d159679 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JPipelineStepRunner.java +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JPipelineStepRunner.java @@ -13,8 +13,9 @@ * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ -package ai.konduit.serving.deeplearning4j.step; +package ai.konduit.serving.models.deeplearning4j.step; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.data.NDArray; import ai.konduit.serving.pipeline.api.exception.ModelLoadingException; @@ -89,7 +90,7 @@ public PipelineStep getPipelineStep() { } @Override - public Data exec(Data data) { + public Data exec(Context ctx, Data data) { //First: Get array //TODO HANDLE DIFFERENT NAMES (Not hardcoded) diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java similarity index 96% rename from konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java rename to konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java index 945c71e35..8b2a06646 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/java/ai/konduit/serving/models/deeplearning4j/step/DL4JPipelineStepRunnerFactory.java @@ -13,7 +13,7 @@ * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ -package ai.konduit.serving.deeplearning4j.step; +package ai.konduit.serving.models.deeplearning4j.step; import ai.konduit.serving.pipeline.api.step.PipelineStep; import ai.konduit.serving.pipeline.api.step.PipelineStepRunner; diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.serde.JsonSubTypesMapping b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.serde.JsonSubTypesMapping index d207284fe..3335c11b2 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.serde.JsonSubTypesMapping +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.serde.JsonSubTypesMapping @@ -14,4 +14,4 @@ # SPDX-License-Identifier: Apache-2.0 ################################################################################ -ai.konduit.serving.deeplearning4j.serde.DL4JJsonSubTypesMapping \ No newline at end of file +ai.konduit.serving.models.deeplearning4j.serde.DL4JJsonSubTypesMapping \ No newline at end of file diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.step.PipelineStepRunnerFactory b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.step.PipelineStepRunnerFactory index fe93ae4a3..8e9bb990b 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.step.PipelineStepRunnerFactory +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.step.PipelineStepRunnerFactory @@ -14,4 +14,4 @@ # SPDX-License-Identifier: Apache-2.0 ################################################################################ -ai.konduit.serving.deeplearning4j.step.DL4JPipelineStepRunnerFactory \ No newline at end of file +ai.konduit.serving.models.deeplearning4j.step.DL4JPipelineStepRunnerFactory \ No newline at end of file diff --git a/konduit-serving-models/konduit-serving-deeplearning4j/src/test/java/ai/konduit/serving/deeplearning4j/TestDL4JModelStep.java b/konduit-serving-models/konduit-serving-deeplearning4j/src/test/java/ai/konduit/serving/deeplearning4j/TestDL4JModelStep.java index af5366697..9eb451be5 100644 --- a/konduit-serving-models/konduit-serving-deeplearning4j/src/test/java/ai/konduit/serving/deeplearning4j/TestDL4JModelStep.java +++ b/konduit-serving-models/konduit-serving-deeplearning4j/src/test/java/ai/konduit/serving/deeplearning4j/TestDL4JModelStep.java @@ -15,7 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.deeplearning4j; -import ai.konduit.serving.deeplearning4j.step.DL4JModelPipelineStep; +import ai.konduit.serving.models.deeplearning4j.step.DL4JModelPipelineStep; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.data.NDArray; import ai.konduit.serving.pipeline.api.pipeline.Pipeline; @@ -73,21 +73,21 @@ public void testSimpleMLN() throws Exception { Data d = Data.singleton("in", NDArray.create(arr)); - Data out = e.exec(d); - INDArray actual = (INDArray) out.getNDArray(outName).get(); //TODO NO CAST + Data out = e.exec(null, d); + INDArray actual = out.getNDArray(outName).getAs(INDArray.class); assertEquals(exp, actual); String json = p.toJson(); System.out.println(json); Pipeline pJson = Pipeline.fromJson(json); - INDArray outJson = (INDArray) pJson.executor().exec(d).getNDArray(outName).get(); //TODO NO CAST + INDArray outJson = pJson.executor().exec(null, d).getNDArray(outName).getAs(INDArray.class); assertEquals(exp, outJson); String yaml = p.toYaml(); System.out.println(yaml); Pipeline pYaml = Pipeline.fromYaml(yaml); - INDArray outYaml = (INDArray) pYaml.executor().exec(d).getNDArray(outName).get(); //TODO NO CAST + INDArray outYaml = pYaml.executor().exec(null, d).getNDArray(outName).getAs(INDArray.class); assertEquals(exp, outYaml); } } @@ -113,22 +113,22 @@ public void testSimpleCompGraph() throws Exception { INDArray arr = Nd4j.rand(DataType.FLOAT, 3, 4); INDArray exp = predictFromFileCG(netFile, arr)[0]; - Data d = Data.singleton("in", NDArray.create(arr)); //TODO NO CAST + Data d = Data.singleton("in", NDArray.create(arr)); - Data out = e.exec(d); - INDArray actual = (INDArray) out.getNDArray(outName).get(); //TODO NO CAST + Data out = e.exec(null, d); + INDArray actual = out.getNDArray(outName).getAs(INDArray.class); assertEquals(exp, actual); String json = p.toJson(); Pipeline pJson = Pipeline.fromJson(json); - INDArray outJson = (INDArray) pJson.executor().exec(d).getNDArray(outName).get(); //TODO NO CAST + INDArray outJson = pJson.executor().exec(null, d).getNDArray(outName).getAs(INDArray.class); assertEquals(exp, outJson); String yaml = p.toYaml(); Pipeline pYaml = Pipeline.fromYaml(yaml); - INDArray outYaml = (INDArray) pYaml.executor().exec(d).getNDArray(outName).get(); //TODO NO CAST + INDArray outYaml = pYaml.executor().exec(null, d).getNDArray(outName).getAs(INDArray.class); assertEquals(exp, outYaml); } } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Context.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Context.java new file mode 100644 index 000000000..8e56438bf --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Context.java @@ -0,0 +1,35 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.api.context; + + +/** + * Context is used for features such as profiling and metrics. + * + * See {@link Profiler} and {@link Metrics} for more details + * + * @author Alex Black + */ +public interface Context { + + Metrics metrics(); + + Profiler profiler(); + +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Metrics.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Metrics.java new file mode 100644 index 000000000..c130a16b4 --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Metrics.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.api.context; + +/** + * Metrics are used to record values, for debugging and visualization. + * + * Instances of the Metrics interface are available within a PipelineStepRunner's exec method via the {@link Context#metrics()} + * method. + * + * @author Alex Black + */ +public interface Metrics { + + + +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Profiler.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Profiler.java new file mode 100644 index 000000000..10fd2be60 --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/context/Profiler.java @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.api.context; + +/** + * The Profiler interface is used within each PipelineStep (technically, within each {@link ai.konduit.serving.pipeline.api.step.PipelineStepRunner}) + * for performance analysis and debugging purposes. + * Specifically, for each event within a PipelineStepRunner such as "load data", "convert input array", etc - developers + * can profile events based on their start and end times.
+ * Events are identified by a String key, and have both start and end times. + * + *

+ * For example: + * 

+ * {@code
+ * @Override
+ * public Data exec(Context ctx, Data data){
+ *     Profiler p = ctx.profiler();
+ *     p.eventStart("DataConversion");
+ *     data.getNDArray("myArray").getAs(float[][].class);
+ *     p.eventEnd("DataConversion");
+ *     ...
+ * }}
+ *

+ * If the profiler is not enabled globally for the Pipeline execution, profiler calls are a no-op hence have virtually + * no overhead. + *

+ * Profiler eventStart/End calls can be nested and events need not end before earlier ones start - for example, both + * {@code eventStart("X"), eventStart("Y"), eventEnd("Y"), eventEnd("X")} and + * {@code eventStart("X"), eventStart("Y"), eventEnd("X"), eventEnd("Y")} are valid. However, + * {@code eventEnd("X"), eventStart("X")} is not valid (end before start), which wil log a warning. + *

+ * Note that every eventStart call should have a corresponding eventEnd call some time before the PipelineStepRunner.exec + * method returns. Any profiler eventStart calls without a corresponding eventEnd call will have eventEnd called once + * the exec method returns. + *

+ * Event keys are usually the same on all calls of a given PipelineStepRunner's exec method, but this need not be the case + * in general + * + * @author Alex Black + */ +public interface Profiler { + + /** + * Returns true if the profiler is enabled globally, or false otherwise + */ + boolean profilerEnabled(); + + /** + * Start the timer for the event with the specified key. + */ + void eventStart(String key); + + /** + * End the timer for the event with the specified key. + * Should be called some time after {@link #eventStart(String)} + */ + void eventEnd(String key); + +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Data.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Data.java index fa027a633..f429085d1 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Data.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Data.java @@ -18,7 +18,8 @@ package ai.konduit.serving.pipeline.api.data; import ai.konduit.serving.pipeline.impl.data.*; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; import ai.konduit.serving.pipeline.impl.serde.DataJsonDeserializer; import ai.konduit.serving.pipeline.impl.serde.DataJsonSerializer; import ai.konduit.serving.pipeline.util.ObjectMappers; @@ -172,10 +173,19 @@ static boolean equals(@NonNull Data d1, @NonNull Data d2){ ValueType vt = d1.type(s); switch (vt){ case LIST: - case IMAGE: default: //TODO throw new UnsupportedOperationException(vt + " equality not yet implemented"); + case IMAGE: + Png png1 = d1.getImage(s).getAs(Png.class); + Png png2 = d1.getImage(s).getAs(Png.class); + + byte[] pngBytes1 = png1.getBytes(); + byte[] pngBytes2 = png2.getBytes(); + + if(!Arrays.equals(pngBytes1, pngBytes2)) + return false; + break; case NDARRAY: //TODO this is inefficient - but robust... NDArray a1 = d1.getNDArray(s); diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Image.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Image.java index dc9ac3a25..832259b99 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Image.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/Image.java @@ -18,8 +18,11 @@ package ai.konduit.serving.pipeline.api.data; +import ai.konduit.serving.pipeline.api.format.ImageFactory; import ai.konduit.serving.pipeline.api.format.ImageFormat; import ai.konduit.serving.pipeline.registry.ImageFactoryRegistry; +import lombok.NonNull; +import org.nd4j.common.base.Preconditions; public interface Image { @@ -27,11 +30,22 @@ public interface Image { T getAs(ImageFormat format); - static Image create(Object from){ - throw new UnsupportedOperationException("Not yet implemented"); + T getAs(Class type); + + boolean canGetAs(ImageFormat format); + + boolean canGetAs(Class type); + + //TODO how will this work for PNG, JPG etc files? + static Image create(@NonNull Object from){ + ImageFactory f = ImageFactoryRegistry.getFactoryFor(from); + Preconditions.checkState(f != null, "Unable to create Image from object of %s - no ImageFactory instances" + + " are available that can convert this type to Konduit Serving Image", from.getClass()); + + return f.create(from); } - static boolean canCreateFrom(Object from){ + static boolean canCreateFrom(@NonNull Object from){ return ImageFactoryRegistry.getFactoryFor(from) != null; } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/ValueType.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/ValueType.java new file mode 100644 index 000000000..603e66346 --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/ValueType.java @@ -0,0 +1,30 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ +package ai.konduit.serving.pipeline.api.data; + +public enum ValueType { + NDARRAY, + STRING, + BYTES, + IMAGE, + DOUBLE, + INT64, + BOOLEAN, + DATA, + LIST +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataConversionException.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataConversionException.java new file mode 100644 index 000000000..66ca60ceb --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataConversionException.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.api.exception; + +public class DataConversionException extends RuntimeException { + public DataConversionException() { + } + + public DataConversionException(String message) { + super(message); + } + + public DataConversionException(String message, Throwable cause) { + super(message, cause); + } + + public DataConversionException(Throwable cause) { + super(cause); + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataLoadingException.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataLoadingException.java new file mode 100644 index 000000000..6488bd26d --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/exception/DataLoadingException.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.api.exception; + +public class DataLoadingException extends RuntimeException { + + public DataLoadingException() { + } + + public DataLoadingException(String message) { + super(message); + } + + public DataLoadingException(String message, Throwable cause) { + super(message, cause); + } + + public DataLoadingException(Throwable cause) { + super(cause); + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/ImageConverter.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/ImageConverter.java index b05318274..158f7a605 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/ImageConverter.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/ImageConverter.java @@ -22,8 +22,11 @@ public interface ImageConverter { - boolean canConvert(Image from, ImageFormat to); + boolean canConvert(Image from, ImageFormat to); - Image convert(Image from, ImageFormat to); + boolean canConvert(Image from, Class to); + T convert(Image from, ImageFormat to); + + T convert(Image from, Class to); } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/NDArrayConverter.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/NDArrayConverter.java index f6933c14f..3a90107ef 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/NDArrayConverter.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/format/NDArrayConverter.java @@ -23,7 +23,7 @@ public interface NDArrayConverter { - boolean canConvert(NDArray from, NDArrayFormat to); + boolean canConvert(NDArray from, NDArrayFormat to); boolean canConvert(NDArray from, Class to); diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/pipeline/PipelineExecutor.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/pipeline/PipelineExecutor.java index 735fd4ac6..73cd0d55b 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/pipeline/PipelineExecutor.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/pipeline/PipelineExecutor.java @@ -15,6 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.api.pipeline; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.step.PipelineStepRunner; import org.slf4j.Logger; @@ -24,6 +25,8 @@ /** * A pipeline executor implements the actual execution behind the {@link Data} -> {@link Data} mapping that is defined * by a {@link Pipeline} + * + * @author Alex Black */ public interface PipelineExecutor { @@ -40,15 +43,15 @@ public interface PipelineExecutor { /** * Execute the pipeline on the specified Data instance */ - Data exec(Data data); + Data exec(Context context, Data data); /** * Execute the pipeline on all of the specified Data instances */ - default Data[] exec(Data... data) { + default Data[] exec(Context context, Data... data) { Data[] out = new Data[data.length]; for (int i = 0; i < data.length; i++) { - out[i] = exec(data[i]); + out[i] = exec(context, data[i]); } return out; } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/step/PipelineStepRunner.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/step/PipelineStepRunner.java index a98eac503..8f3407e79 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/step/PipelineStepRunner.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/step/PipelineStepRunner.java @@ -15,6 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.api.step; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import java.io.Closeable; @@ -42,15 +43,15 @@ public interface PipelineStepRunner extends Closeable { /** * Execute the pipeline on the specified Data instance */ - Data exec(Data data); + Data exec(Context ctx, Data data); /** * Execute the pipeline on all of the specified Data instances */ - default Data[] exec(Data... data) { + default Data[] exec(Context ctx, Data... data) { Data[] out = new Data[data.length]; for (int i = 0; i < data.length; i++) { - out[i] = exec(data[i]); + out[i] = exec(ctx, data[i]); } return out; } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/JData.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/JData.java index 8bd7e9986..c0402477c 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/JData.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/JData.java @@ -18,6 +18,7 @@ import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.data.NDArray; +import ai.konduit.serving.pipeline.api.data.ValueType; import ai.konduit.serving.pipeline.impl.data.wrappers.*; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/Value.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/Value.java index 2284d5399..36437425c 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/Value.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/Value.java @@ -15,6 +15,8 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.data; +import ai.konduit.serving.pipeline.api.data.ValueType; + public interface Value { ValueType type(); diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ValueType.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ValueType.java deleted file mode 100644 index 5bc204d99..000000000 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ValueType.java +++ /dev/null @@ -1,28 +0,0 @@ -/* ****************************************************************************** - * Copyright (c) 2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ -package ai.konduit.serving.pipeline.impl.data; - -public enum ValueType { - NDARRAY, - STRING, - BYTES, - IMAGE, - DOUBLE, - INT64, - BOOLEAN, - DATA, - LIST -} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BImage.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BImage.java new file mode 100644 index 000000000..373b2d9be --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BImage.java @@ -0,0 +1,27 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.data.image; + +import java.awt.image.BufferedImage; + +public class BImage extends BaseImage { + public BImage(BufferedImage image) { + super(image); + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BaseImage.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BaseImage.java new file mode 100644 index 000000000..b7dc04a67 --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/BaseImage.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.data.image; + +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.format.ImageConverter; +import ai.konduit.serving.pipeline.api.format.ImageFormat; +import ai.konduit.serving.pipeline.registry.ImageConverterRegistry; +import lombok.AllArgsConstructor; +import org.nd4j.common.base.Preconditions; + +import java.util.Arrays; + +@AllArgsConstructor +public abstract class BaseImage implements Image { + + private final T image; + + @Override + public Object get() { + return image; + } + + @Override + public T getAs(ImageFormat format) { + return ImageConverterRegistry.getConverterFor(this, format).convert(this, format); + } + + @Override + public T getAs(Class type) { + ImageConverter converter = ImageConverterRegistry.getConverterFor(this, type); + Preconditions.checkState(converter != null, "No converter found for converting from %s to %s", image.getClass(), type); + return converter.convert(this, type); + } + + @Override + public boolean canGetAs(ImageFormat format) { + ImageConverter converter = ImageConverterRegistry.getConverterFor(this, format); + return converter != null; + } + + @Override + public boolean canGetAs(Class type) { + return false; + } + + @Override + public boolean equals(Object o){ + if(!(o instanceof Image)) + return false; + + Image o2 = (Image)o; + + //TODO is this actually reliable for checks? + Png png1 = getAs(Png.class); + Png png2 = o2.getAs(Png.class); + + byte[] b1 = png1.getBytes(); + byte[] b2 = png2.getBytes(); + + return Arrays.equals(b1, b2); + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/Png.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/Png.java new file mode 100644 index 000000000..d2b6e8906 --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/Png.java @@ -0,0 +1,74 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.data.image; + +import ai.konduit.serving.pipeline.api.exception.DataLoadingException; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.commons.io.FileUtils; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +@Data +@AllArgsConstructor +public class Png { + + private ByteBuffer pngFileBytes; + + public Png(File file){ + try { + pngFileBytes = ByteBuffer.wrap(FileUtils.readFileToByteArray(file)); + } catch (IOException e){ + throw new DataLoadingException("Unable to load PNG image from file " + file.getAbsolutePath()); + } + } + + public Png(byte[] bytes){ + this.pngFileBytes = ByteBuffer.wrap(bytes); + } + + public byte[] getBytes(){ + if(pngFileBytes.hasArray()){ + return pngFileBytes.array(); + } else { + byte[] bytes = new byte[pngFileBytes.capacity()]; + pngFileBytes.position(0); + pngFileBytes.get(bytes); + return bytes; + } + + } + + public void save(File f) throws IOException { + FileUtils.writeByteArrayToFile(f, getBytes()); + } + + public void write(OutputStream os) throws IOException { + boolean buffered = os instanceof BufferedOutputStream; + if(!buffered) + os = new BufferedOutputStream(os); + try(OutputStream o = os){ + o.write(getBytes()); + } + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaImage.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/PngImage.java similarity index 83% rename from konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaImage.java rename to konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/PngImage.java index 9e6e65880..bf9a14157 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaImage.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/image/PngImage.java @@ -16,7 +16,11 @@ * ***************************************************************************** */ -package ai.konduit.serving.pipeline.impl.data.java; +package ai.konduit.serving.pipeline.impl.data.image; -public class JavaImage { +public class PngImage extends BaseImage { + + public PngImage(Png image) { + super(image); + } } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/BaseNDArray.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ndarray/BaseNDArray.java similarity index 92% rename from konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/BaseNDArray.java rename to konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ndarray/BaseNDArray.java index f62ad205e..1378c7d28 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/api/data/BaseNDArray.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ndarray/BaseNDArray.java @@ -16,17 +16,17 @@ * ***************************************************************************** */ -package ai.konduit.serving.pipeline.api.data; +package ai.konduit.serving.pipeline.impl.data.ndarray; +import ai.konduit.serving.pipeline.api.data.NDArray; import ai.konduit.serving.pipeline.api.format.NDArrayConverter; import ai.konduit.serving.pipeline.api.format.NDArrayFormat; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; import ai.konduit.serving.pipeline.registry.NDArrayConverterRegistry; import lombok.AllArgsConstructor; import org.nd4j.common.base.Preconditions; @AllArgsConstructor -public class BaseNDArray implements NDArray { +public abstract class BaseNDArray implements NDArray { private final T array; @@ -42,7 +42,6 @@ public T getAs(NDArrayFormat format) { @Override public T getAs(Class type) { - //TODO check to avoid NPE NDArrayConverter converter = NDArrayConverterRegistry.getConverterFor(this, type); Preconditions.checkState(converter != null, "No converter found for converting from %s to %s", array.getClass(), type); return converter.convert(this, type); diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/SerializedNDArray.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ndarray/SerializedNDArray.java similarity index 97% rename from konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/SerializedNDArray.java rename to konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ndarray/SerializedNDArray.java index 6cc52ce08..a79086aa1 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/SerializedNDArray.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/ndarray/SerializedNDArray.java @@ -16,7 +16,7 @@ * ***************************************************************************** */ -package ai.konduit.serving.pipeline.impl.format; +package ai.konduit.serving.pipeline.impl.data.ndarray; import ai.konduit.serving.pipeline.api.data.NDArrayType; import lombok.AllArgsConstructor; diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BaseValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BaseValue.java index 62d3f6acf..a3ba09fd9 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BaseValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BaseValue.java @@ -19,7 +19,7 @@ package ai.konduit.serving.pipeline.impl.data.wrappers; import ai.konduit.serving.pipeline.impl.data.Value; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; import lombok.AllArgsConstructor; @AllArgsConstructor diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BooleanValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BooleanValue.java index e53f0400e..054bd52d5 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BooleanValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BooleanValue.java @@ -15,7 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.data.wrappers; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class BooleanValue extends BaseValue { public BooleanValue(Boolean value) { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BytesValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BytesValue.java index 64b4680c9..d6d290656 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BytesValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/BytesValue.java @@ -15,7 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.data.wrappers; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class BytesValue extends BaseValue { public BytesValue(byte[] value) { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DataValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DataValue.java index 5615cf83e..3a032266b 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DataValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DataValue.java @@ -19,7 +19,7 @@ package ai.konduit.serving.pipeline.impl.data.wrappers; import ai.konduit.serving.pipeline.api.data.Data; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class DataValue extends BaseValue { public DataValue(Data value) { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DoubleValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DoubleValue.java index 59c984d98..0dc210b29 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DoubleValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/DoubleValue.java @@ -15,7 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.data.wrappers; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class DoubleValue extends BaseValue { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/ImageValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/ImageValue.java index 8ddd9c036..2a8345680 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/ImageValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/ImageValue.java @@ -16,7 +16,7 @@ package ai.konduit.serving.pipeline.impl.data.wrappers; import ai.konduit.serving.pipeline.api.data.Image; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class ImageValue extends BaseValue { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/IntValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/IntValue.java index 87e57ae73..d3400e3d2 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/IntValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/IntValue.java @@ -15,7 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.data.wrappers; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class IntValue extends BaseValue { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/NDArrayValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/NDArrayValue.java index 83044a796..8ee8a1e5c 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/NDArrayValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/NDArrayValue.java @@ -16,9 +16,7 @@ package ai.konduit.serving.pipeline.impl.data.wrappers; import ai.konduit.serving.pipeline.api.data.NDArray; -import ai.konduit.serving.pipeline.impl.data.Value; -import ai.konduit.serving.pipeline.impl.data.ValueType; -import lombok.AllArgsConstructor; +import ai.konduit.serving.pipeline.api.data.ValueType; public class NDArrayValue extends BaseValue { public NDArrayValue(NDArray value) { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/StringValue.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/StringValue.java index 4ecd5a10f..06f4010c9 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/StringValue.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/wrappers/StringValue.java @@ -15,7 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.data.wrappers; -import ai.konduit.serving.pipeline.impl.data.ValueType; +import ai.konduit.serving.pipeline.api.data.ValueType; public class StringValue extends BaseValue { public StringValue(String value) { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageConverters.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageConverters.java new file mode 100644 index 000000000..92217ae9e --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageConverters.java @@ -0,0 +1,132 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.format; + +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.exception.DataConversionException; +import ai.konduit.serving.pipeline.api.exception.DataLoadingException; +import ai.konduit.serving.pipeline.api.format.ImageConverter; +import ai.konduit.serving.pipeline.api.format.ImageFormat; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import lombok.AllArgsConstructor; +import org.nd4j.common.base.Preconditions; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +public class JavaImageConverters { + + private JavaImageConverters(){ } + + public static class IdentityConverter implements ImageConverter { + + @Override + public boolean canConvert(Image from, ImageFormat to) { + return false; + } + + @Override + public boolean canConvert(Image from, Class to) { + return to.isAssignableFrom(from.get().getClass()); + } + + @Override + public T convert(Image from, ImageFormat to) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public T convert(Image from, Class to) { + Preconditions.checkState(canConvert(from, to), "Unable to convert %s to %s", from.get().getClass(), to); + return (T)from.get(); + } + } + + @AllArgsConstructor + public static abstract class BaseConverter implements ImageConverter { + protected Class cFrom; + protected Class cTo; + + @Override + public boolean canConvert(Image from, ImageFormat to) { + return false; + } + + @Override + public boolean canConvert(Image from, Class to) { + return cFrom.isAssignableFrom(from.get().getClass()) && cTo.isAssignableFrom(to); + } + + @Override + public T convert(Image from, ImageFormat to) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public T convert(Image from, Class to) { + Preconditions.checkState(canConvert(from, to), "Unable to convert image to format %s", to); + return doConversion(from, to); + } + + protected abstract T doConversion(Image from, Class to); + } + + public static class PngToBufferedImageConverter extends BaseConverter { + + public PngToBufferedImageConverter() { + super(Png.class, BufferedImage.class); + } + + @Override + protected T doConversion(Image from, Class to) { + Png png = (Png) from.get(); + byte[] bytes = png.getBytes(); + try(ByteArrayInputStream is = new ByteArrayInputStream(bytes)){ + BufferedImage bi = ImageIO.read(is); + return (T) bi; + } catch (IOException e){ + throw new DataLoadingException("Error converting PNG to BufferedImage", e); + } + } + } + + public static class BufferedImageToPngConverter extends BaseConverter { + public BufferedImageToPngConverter() { + super(BufferedImage.class, Png.class); + } + + @Override + protected T doConversion(Image from, Class to) { + BufferedImage bi = (BufferedImage) from.get(); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try { + ImageIO.write(bi, "png", os); + } catch (IOException e){ + throw new DataConversionException("Error converting BufferedImage to PNG", e); + } + byte[] bytes = os.toByteArray(); + return (T) new Png(bytes); + } + } + +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageFactory.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageFactory.java new file mode 100644 index 000000000..469a2052d --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaImageFactory.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.format; + +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.exception.DataLoadingException; +import ai.konduit.serving.pipeline.api.format.ImageFactory; +import ai.konduit.serving.pipeline.impl.data.image.BImage; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.data.image.PngImage; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class JavaImageFactory implements ImageFactory { + + private static final Set> supportedClasses = new HashSet<>(); + static { + Set> s = supportedClasses; + s.add(File.class); + s.add(Path.class); + s.add(Png.class); + s.add(BufferedImage.class); + } + + @Override + public Set> supportedTypes() { + return Collections.unmodifiableSet(supportedClasses); + } + + @Override + public boolean canCreateFrom(Object o) { + //TODO what about interfaces, subtypes etc? + return supportedClasses.contains(o.getClass()); + } + + @Override + public Image create(Object o) { + if(o instanceof File || o instanceof Path){ + //Try to infer file type from + File f; + if(o instanceof File){ + f = (File) o; + } else { + f = ((Path)o).toFile(); + } + String name = f.getName(); + if(name.toLowerCase().endsWith(".png")){ + return new PngImage(new Png(f)); + } + throw new DataLoadingException("Unable to create Image object: unable to guess image file format from File" + + " path/filename, or format not supported - " + f.getAbsolutePath()); + } else if(o instanceof Png){ + return new PngImage((Png) o); + } else if(o instanceof BufferedImage){ + return new BImage((BufferedImage) o); + } else { + throw new UnsupportedOperationException("Unable to create Image from object of type: " + o.getClass()); + } + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayConverters.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayConverters.java index 03b50561c..dd0a163e2 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayConverters.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayConverters.java @@ -20,6 +20,7 @@ import ai.konduit.serving.pipeline.api.data.NDArray; import ai.konduit.serving.pipeline.api.data.NDArrayType; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; import ai.konduit.serving.pipeline.api.format.NDArrayConverter; import ai.konduit.serving.pipeline.api.format.NDArrayFormat; import lombok.AllArgsConstructor; @@ -37,7 +38,7 @@ public static class IdentityConverter implements NDArrayConverter { @Override - public boolean canConvert(NDArray from, NDArrayFormat to) { + public boolean canConvert(NDArray from, NDArrayFormat to) { return false; } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaNDArrayFactory.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayFactory.java similarity index 95% rename from konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaNDArrayFactory.java rename to konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayFactory.java index 56cd82bd9..27b69f249 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaNDArrayFactory.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrayFactory.java @@ -16,11 +16,11 @@ * ***************************************************************************** */ -package ai.konduit.serving.pipeline.impl.data.java; +package ai.konduit.serving.pipeline.impl.format; import ai.konduit.serving.pipeline.api.data.NDArray; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; import ai.konduit.serving.pipeline.api.format.NDArrayFactory; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; import java.util.HashSet; import java.util.Set; diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaNDArrays.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrays.java similarity index 86% rename from konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaNDArrays.java rename to konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrays.java index 88dfe055c..c0f2f27f7 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/data/java/JavaNDArrays.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/format/JavaNDArrays.java @@ -16,12 +16,10 @@ * ***************************************************************************** */ -package ai.konduit.serving.pipeline.impl.data.java; +package ai.konduit.serving.pipeline.impl.format; -import ai.konduit.serving.pipeline.api.data.BaseNDArray; -import ai.konduit.serving.pipeline.api.data.NDArray; -import ai.konduit.serving.pipeline.api.format.NDArrayFormat; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; +import ai.konduit.serving.pipeline.impl.data.ndarray.BaseNDArray; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; public class JavaNDArrays { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/pipeline/SequencePipelineExecutor.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/pipeline/SequencePipelineExecutor.java index 89015a6c4..d9812ba8b 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/pipeline/SequencePipelineExecutor.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/pipeline/SequencePipelineExecutor.java @@ -15,6 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.pipeline; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.pipeline.Pipeline; import ai.konduit.serving.pipeline.api.pipeline.PipelineExecutor; @@ -97,10 +98,10 @@ public List getRunners() { } @Override - public Data exec(Data data) { + public Data exec(Context ctx, Data data) { Data current = data; for(PipelineStepRunner psr : runners){ - current = psr.exec(current); + current = psr.exec(ctx, current); } return current; } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonDeserializer.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonDeserializer.java index cdaaceb4f..f5db2519b 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonDeserializer.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonDeserializer.java @@ -19,10 +19,12 @@ package ai.konduit.serving.pipeline.impl.serde; import ai.konduit.serving.pipeline.api.data.Data; +import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.data.NDArray; import ai.konduit.serving.pipeline.api.data.NDArrayType; import ai.konduit.serving.pipeline.impl.data.JData; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; import org.nd4j.shade.jackson.core.JsonParser; import org.nd4j.shade.jackson.core.JsonProcessingException; import org.nd4j.shade.jackson.databind.DeserializationContext; @@ -108,7 +110,14 @@ public Data deserialize(JsonParser jp, JsonNode n) { d.put(s, NDArray.create(ndArray)); } else if (n2.has(Data.RESERVED_KEY_IMAGE_DATA)) { //Image - throw new UnsupportedOperationException("Image deserialization not yet implemented"); + String format = n2.get(Data.RESERVED_KEY_IMAGE_FORMAT).textValue(); + if(!"PNG".equalsIgnoreCase(format)){ + throw new UnsupportedOperationException("Deserialization of formats other than PNG not yet implemented"); + } + String base64Data = n2.get(Data.RESERVED_KEY_IMAGE_DATA).textValue(); + byte[] bytes = Base64.getDecoder().decode(base64Data); + Png png = new Png(bytes); + d.put(s, Image.create(png)); } else { //Must be data Data dInner = deserialize(jp, n2); diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonSerializer.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonSerializer.java index 2553932c4..c42328250 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonSerializer.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/serde/DataJsonSerializer.java @@ -22,8 +22,9 @@ import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.data.NDArray; import ai.konduit.serving.pipeline.api.data.NDArrayType; -import ai.konduit.serving.pipeline.impl.data.ValueType; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; +import ai.konduit.serving.pipeline.api.data.ValueType; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; import org.nd4j.shade.jackson.core.JsonGenerator; import org.nd4j.shade.jackson.databind.JsonSerializer; import org.nd4j.shade.jackson.databind.ObjectMapper; @@ -134,7 +135,15 @@ private void writeLong(JsonGenerator jg, long l) throws IOException { } private void writeImage(JsonGenerator jg, Image i) throws IOException { - throw new UnsupportedOperationException("Not yet implemented: writing image JSON"); + Png png = i.getAs(Png.class); + byte[] imgData = png.getBytes(); + jg.writeStartObject(); + jg.writeFieldName(Data.RESERVED_KEY_IMAGE_FORMAT); + jg.writeString("PNG"); //TODO No magic constant + jg.writeFieldName(Data.RESERVED_KEY_IMAGE_DATA); + String base64 = Base64.getEncoder().encodeToString(imgData); + jg.writeString(base64); + jg.writeEndObject(); } private void writeNDArray(JsonGenerator jg, NDArray n) throws IOException { diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/step/logging/LoggingPipelineStepRunner.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/step/logging/LoggingPipelineStepRunner.java index 38356174f..c0bacc985 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/step/logging/LoggingPipelineStepRunner.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/impl/step/logging/LoggingPipelineStepRunner.java @@ -15,6 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.step.logging; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.step.PipelineStep; import ai.konduit.serving.pipeline.api.step.PipelineStepRunner; @@ -52,7 +53,7 @@ public PipelineStep getPipelineStep() { } @Override - public Data exec(Data data) { + public Data exec(Context ctx, Data data) { Level logLevel = step.getLogLevel(); LoggingPipelineStep.Log toLog = step.getLog(); boolean keysOnly = toLog == LoggingPipelineStep.Log.KEYS; diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/AbstractRegistry.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/AbstractRegistry.java index eeec34cbe..e52851c5f 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/AbstractRegistry.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/AbstractRegistry.java @@ -91,4 +91,10 @@ protected synchronized void init(){ factoriesMap = m; factories = l; } + + public void addFactoryInstance(T factory){ + if(factories == null) + init(); + this.factories.add(factory); + } } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverteRegistry.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverteRegistry.java deleted file mode 100644 index 1eaf448b9..000000000 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverteRegistry.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * ****************************************************************************** - * * Copyright (c) 2020 Konduit K.K. - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package ai.konduit.serving.pipeline.registry; - -import ai.konduit.serving.pipeline.api.data.Image; -import ai.konduit.serving.pipeline.api.format.ImageConverter; -import ai.konduit.serving.pipeline.api.format.ImageFormat; -import lombok.NonNull; -import org.nd4j.common.primitives.Pair; - -import java.util.Collections; -import java.util.List; -import java.util.Set; - -public class ImageConverteRegistry extends AbstractRegistry { - - private static final ImageConverteRegistry INSTANCE = new ImageConverteRegistry(); - - protected ImageConverteRegistry(){ - super(ImageConverter.class); - } - - public static int numFactories(){ - return INSTANCE.registryNumFactories(); - } - - public static List getFactories(){ - return INSTANCE.registryGetFactories(); - } - - public static ImageConverter getFactoryFor(@NonNull Object o){ - return INSTANCE.registryGetFactoryFor(o); - } - - @Override - public boolean acceptFactory(ImageConverter factory, Object o) { - Pair p = (Pair) o; - return factory.canConvert(p.getFirst(), p.getSecond()); - } - - @Override - public Set> supportedForFactory(ImageConverter factory) { - return Collections.emptySet(); - } -} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverterRegistry.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverterRegistry.java new file mode 100644 index 000000000..96b81f777 --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageConverterRegistry.java @@ -0,0 +1,144 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.registry; + +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.data.NDArray; +import ai.konduit.serving.pipeline.api.format.*; +import ai.konduit.serving.pipeline.api.format.ImageConverter; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import lombok.AllArgsConstructor; +import lombok.NonNull; +import org.nd4j.common.primitives.Pair; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class ImageConverterRegistry extends AbstractRegistry { + + private static final ImageConverterRegistry INSTANCE = new ImageConverterRegistry(); + + protected ImageConverterRegistry(){ + super(ImageConverter.class); + } + + public static int numFactories(){ + return INSTANCE.registryNumFactories(); + } + + public static List getFactories(){ + return INSTANCE.registryGetFactories(); + } + + public static ImageConverter getFactoryFor(@NonNull Object o){ + return INSTANCE.registryGetFactoryFor(o); + } + + @Override + public boolean acceptFactory(ImageConverter factory, Object o) { + Pair p = (Pair) o; + return factory.canConvert(p.getFirst(), p.getSecond()); + } + + @Override + public Set> supportedForFactory(ImageConverter factory) { + return Collections.emptySet(); + } + + public static ImageConverter getConverterFor(Image img, Class type ){ + return INSTANCE.getConverterForClass(img, type); + } + + public static ImageConverter getConverterFor(Image img, ImageFormat type ){ + return INSTANCE.getConverterForType(img, type); + } + + public ImageConverter getConverterForClass(Image img, Class type ){ + if(factories == null) + init(); + + if(factoriesMap.containsKey(type)){ + return factoriesMap.get(type).get(0); //TODO multiple converters available + } + + for(ImageConverter c : factories){ + if(c.canConvert(img, type)){ + return c; + } + } + + //No factory is available. Try to fall back on X -> PNG -> Y + if(type != Png.class){ + ImageConverter c1 = getConverterForClass(img, Png.class); + if(c1 != null){ + Image i2 = Image.create(c1.convert(img, Png.class)); //TODO this is ugly - we throw this result away! + ImageConverter c2 = getConverterForClass(i2, type); + return new TwoStepImageConverter(img.get().getClass(), type, c1, c2); + } + } + + return null; + } + + public ImageConverter getConverterForType(Image img, ImageFormat type ){ + if(factories == null) + init(); + + for(ImageConverter c : factories){ + if(c.canConvert(img, type)){ + return c; + } + } + return null; + } + + public static void addConverter(ImageConverter f){ + INSTANCE.addFactoryInstance(f); + } + + @AllArgsConstructor + private static class TwoStepImageConverter implements ImageConverter { + private Class cFrom; + private Class cTo; + private ImageConverter c1; + private ImageConverter c2; + + @Override + public boolean canConvert(Image from, ImageFormat to) { + return false; + } + + @Override + public boolean canConvert(Image from, Class to) { + return cFrom.isAssignableFrom(from.get().getClass()) && to.isAssignableFrom(cTo); + } + + @Override + public T convert(Image from, ImageFormat to) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public T convert(Image from, Class to) { + Image png = Image.create(c1.convert(from, Png.class)); + return (T) c2.convert(png, cTo); + } + } +} diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageFactoryRegistry.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageFactoryRegistry.java index 92ba47b68..2421af0a4 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageFactoryRegistry.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/ImageFactoryRegistry.java @@ -42,4 +42,8 @@ public static List getFactories(){ public static ImageFactory getFactoryFor(@NonNull Object o){ return INSTANCE.registryGetFactoryFor(o); } + + public static void addFactory(ImageFactory f){ + INSTANCE.addFactoryInstance(f); + } } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayConverterRegistry.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayConverterRegistry.java index d7a5bb81b..1f5c33222 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayConverterRegistry.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayConverterRegistry.java @@ -18,10 +18,12 @@ package ai.konduit.serving.pipeline.registry; +import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.data.NDArray; -import ai.konduit.serving.pipeline.api.format.NDArrayConverter; -import ai.konduit.serving.pipeline.api.format.NDArrayFactory; -import ai.konduit.serving.pipeline.api.format.NDArrayFormat; +import ai.konduit.serving.pipeline.api.format.*; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; +import lombok.AllArgsConstructor; import lombok.NonNull; import org.nd4j.common.primitives.Pair; @@ -81,6 +83,17 @@ public NDArrayConverter getConverterForClass(NDArray arr, Class type ){ return c; } } + + //No factory is available. Try to fall back on X -> SerializedNDArray -> Y + if(type != SerializedNDArray.class){ + NDArrayConverter c1 = getConverterForClass(arr, SerializedNDArray.class); + if(c1 != null){ + NDArray arr2 = NDArray.create(c1.convert(arr, SerializedNDArray.class)); //TODO this is ugly - we throw this result away! + NDArrayConverter c2 = getConverterForClass(arr2, type); + return new TwoStepNDArrayConverter(arr.get().getClass(), type, c1, c2); + } + } + return null; } @@ -95,4 +108,37 @@ public NDArrayConverter getConverterForType(NDArray arr, NDArrayFormat type ) } return null; } + + public static void addConverter(NDArrayConverter f){ + INSTANCE.addFactoryInstance(f); + } + + @AllArgsConstructor + private static class TwoStepNDArrayConverter implements NDArrayConverter { + private Class cFrom; + private Class cTo; + private NDArrayConverter c1; + private NDArrayConverter c2; + + @Override + public boolean canConvert(NDArray from, NDArrayFormat to) { + return false; + } + + @Override + public boolean canConvert(NDArray from, Class to) { + return cFrom.isAssignableFrom(from.get().getClass()) && to.isAssignableFrom(cTo); + } + + @Override + public T convert(NDArray from, NDArrayFormat to) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public T convert(NDArray from, Class to) { + NDArray sArr = NDArray.create(c1.convert(from, SerializedNDArray.class)); + return (T) c2.convert(sArr, cTo); + } + } } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayFactoryRegistry.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayFactoryRegistry.java index 4d4c03a42..046238e46 100644 --- a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayFactoryRegistry.java +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/registry/NDArrayFactoryRegistry.java @@ -18,6 +18,7 @@ package ai.konduit.serving.pipeline.registry; +import ai.konduit.serving.pipeline.api.format.ImageFactory; import ai.konduit.serving.pipeline.api.format.NDArrayFactory; import lombok.NonNull; @@ -42,4 +43,8 @@ public static List getFactories(){ public static NDArrayFactory getFactoryFor(@NonNull Object o){ return INSTANCE.registryGetFactoryFor(o); } + + public static void addFactory(NDArrayFactory f){ + INSTANCE.addFactoryInstance(f); + } } diff --git a/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/util/FileUtils.java b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/util/FileUtils.java new file mode 100644 index 000000000..80bc07d3a --- /dev/null +++ b/konduit-serving-pipeline/src/main/java/ai/konduit/serving/pipeline/util/FileUtils.java @@ -0,0 +1,41 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.util; + +import java.io.File; + +public class FileUtils { + + private FileUtils(){ } + + public static File getTempFileBaseDir(){ + File f = new File(System.getProperty("java.io.tmpdir"), "konduit-serving"); + if(!f.exists()) + f.mkdirs(); + return f; + } + + public static File getTempFileDir(String subdirectory){ + File f = new File(getTempFileBaseDir(), subdirectory); + if(!f.exists()) + f.mkdirs(); + return f; + } + +} diff --git a/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter b/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter new file mode 100644 index 000000000..d234e6632 --- /dev/null +++ b/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageConverter @@ -0,0 +1,21 @@ +# +# /* ****************************************************************************** +# * Copyright (c) 2020 Konduit K.K. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +ai.konduit.serving.pipeline.impl.format.JavaImageConverters$IdentityConverter +ai.konduit.serving.pipeline.impl.format.JavaImageConverters$PngToBufferedImageConverter +ai.konduit.serving.pipeline.impl.format.JavaImageConverters$BufferedImageToPngConverter \ No newline at end of file diff --git a/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory b/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory new file mode 100644 index 000000000..fc72cc94d --- /dev/null +++ b/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.ImageFactory @@ -0,0 +1,20 @@ +# +# /* ****************************************************************************** +# * Copyright (c) 2020 Konduit K.K. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + + +ai.konduit.serving.pipeline.impl.format.JavaImageFactory \ No newline at end of file diff --git a/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.NDArrayFactory b/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.NDArrayFactory index cd3e24f4c..82f3d0e94 100644 --- a/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.NDArrayFactory +++ b/konduit-serving-pipeline/src/main/resources/META-INF/services/ai.konduit.serving.pipeline.api.format.NDArrayFactory @@ -17,4 +17,4 @@ # -ai.konduit.serving.pipeline.impl.data.java.JavaNDArrayFactory \ No newline at end of file +ai.konduit.serving.pipeline.impl.format.JavaNDArrayFactory \ No newline at end of file diff --git a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/DataJsonTest.java b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/DataJsonTest.java index 5c948bfa9..a86795514 100644 --- a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/DataJsonTest.java +++ b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/DataJsonTest.java @@ -19,9 +19,12 @@ package ai.konduit.serving.pipeline.impl.data; import ai.konduit.serving.pipeline.api.data.Data; +import ai.konduit.serving.pipeline.api.data.Image; import ai.konduit.serving.pipeline.api.data.NDArray; +import ai.konduit.serving.pipeline.api.data.ValueType; import org.junit.Ignore; import org.junit.Test; +import org.nd4j.common.resources.Resources; import java.util.Arrays; import java.util.List; @@ -35,7 +38,7 @@ public class DataJsonTest { public void testBasic(){ for(ValueType vt : ValueType.values()){ - if(vt == ValueType.IMAGE || vt == ValueType.LIST ){ //TODO NO WAY TO PUT LISTS INTO DATA YET - WIP TO BE MERGED SOON + if(vt == ValueType.LIST ){ //TODO NO WAY TO PUT LISTS INTO DATA YET - WIP TO BE MERGED SOON System.out.println("SKIPPING: " + vt); continue; } @@ -54,7 +57,7 @@ public void testBasic(){ d = Data.singleton("myKey", new byte[]{0,1,2}); break; case IMAGE: - d = null; + d = Data.singleton("myKey", Image.create(Resources.asFile("data/5_32x32.png"))); break; case DOUBLE: d = Data.singleton("myKey", 1.0); diff --git a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/ImageTests.java b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/ImageTests.java new file mode 100644 index 000000000..c944e93e2 --- /dev/null +++ b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/ImageTests.java @@ -0,0 +1,216 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.data; + +import ai.konduit.serving.pipeline.api.data.Data; +import ai.konduit.serving.pipeline.api.data.Image; +import ai.konduit.serving.pipeline.api.format.ImageConverter; +import ai.konduit.serving.pipeline.api.format.ImageFactory; +import ai.konduit.serving.pipeline.impl.data.image.BaseImage; +import ai.konduit.serving.pipeline.impl.data.image.Png; +import ai.konduit.serving.pipeline.impl.data.image.PngImage; +import ai.konduit.serving.pipeline.impl.format.JavaImageConverters; +import ai.konduit.serving.pipeline.registry.ImageConverterRegistry; +import ai.konduit.serving.pipeline.registry.ImageFactoryRegistry; +import lombok.AllArgsConstructor; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.resources.Resources; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.*; +import java.util.Collections; +import java.util.Set; + +import static org.junit.Assert.*; + +public class ImageTests { + + @Test + public void testBasicLoadingConversion() throws Exception { + + File f = Resources.asFile("data/5_32x32.png"); + System.out.println(f.getAbsolutePath()); + + Image i = Image.create(f); + assertTrue(i instanceof PngImage); + assertTrue(i.get() instanceof Png); + + Png p = i.getAs(Png.class); + + byte[] bytes = p.getBytes(); + byte[] expBytes; + try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ + expBytes = IOUtils.toByteArray(is); + } + + assertArrayEquals(expBytes, bytes); + + + //Data and JSON serialization + Data d = Data.singleton("myImage", i); + String json = d.toJson(); + System.out.println(json); + Data dJson = Data.fromJson(json); + assertEquals(d, dJson); + + + //PNG -> BufferedImage + BufferedImage bi = i.getAs(BufferedImage.class); + BufferedImage biExp = ImageIO.read(f); + boolean eq = bufferedImagesEqual(biExp, bi); + assertTrue("BufferedImage instances should be equal", eq); + + //BufferedImage creation, Data, JSON and BufferedImage -> PNG + Image i2 = Image.create(bi); + Data d2 = Data.singleton("myImage", i2); + String json2 = d2.toJson(); + Data d2Json = Data.fromJson(json2); + + assertEquals(d2, d2Json); + + Png png2 = d2Json.getImage("myImage").getAs(Png.class); + //assertEquals(p, png2); //TODO - this fails - but that doesn't necessarily mean it's a different image given byte[] + boolean eq2 = equalPngs(p, png2); + assertTrue("Images differ after conversion to/from PNG", eq2); + } + + protected static boolean equalPngs(Png png1, Png png2){ + try { + BufferedImage bi1 = ImageIO.read(new ByteArrayInputStream(png1.getBytes())); + BufferedImage bi2 = ImageIO.read(new ByteArrayInputStream(png2.getBytes())); + return bufferedImagesEqual(bi1, bi2); + } catch (Throwable t){ + throw new RuntimeException(t); + } + } + + protected static boolean bufferedImagesEqual(BufferedImage img1, BufferedImage img2) { + if (img1.getHeight() != img2.getHeight() || img1.getWidth() != img2.getWidth()) { + return false; + } + int w = img1.getWidth(); + int h = img1.getHeight(); + + for (int i = 0; i < h; i++) { + for (int j = 0; j < w; j++) { + if (img1.getRGB(i,j) != img2.getRGB(i,j)) { + return false; + } + } + } + return true; + } + + + + @Test + public void test2StepConversion(){ + //The idea: We add some new module with a new image format, X + //In that module, we implement only conversion of X->PNG and PNG->X + //What if we want to do X->Y? + //ImageConverterRegistry will try to do X->PNG->Y - as PNG is something all image types should implement + // conversions to/from for + + File f = Resources.asFile("data/5_32x32.png"); + System.out.println(f.getAbsolutePath()); + + Image i = Image.create(f); + assertTrue(i instanceof PngImage); + assertTrue(i.get() instanceof Png); + Png p = (Png) i.get(); + + + + ImageFactoryRegistry.addFactory(new TestImageFactory()); + ImageConverterRegistry.addConverter(new TIToPng()); + ImageConverterRegistry.addConverter(new PngToTI()); + Image img = Image.create(new TestImageObject(p)); + + //TestImage -> PNG -> BufferedImage + BufferedImage bi = img.getAs(BufferedImage.class); + BufferedImage exp = i.getAs(BufferedImage.class); + assertTrue(bufferedImagesEqual(exp, bi)); + + //BufferedImage -> PNG -> TestImage + Image i2 = Image.create(exp); + TestImageObject out = i2.getAs(TestImageObject.class); + assertTrue(equalPngs(p, out.getPng())); + + } + + @AllArgsConstructor + @lombok.Data + public static class TestImageObject { + private Png png; + } + + public static class TestImage extends BaseImage { + public TestImage(TestImageObject image) { + super(image); + } + } + + public static class TestImageFactory implements ImageFactory { + + @Override + public Set> supportedTypes() { + return Collections.singleton(TestImageObject.class); + } + + @Override + public boolean canCreateFrom(Object o) { + return o instanceof TestImageObject; + } + + @Override + public Image create(Object o) { + Preconditions.checkState(canCreateFrom(o)); + return new TestImage((TestImageObject)o); + } + } + + public static class TIToPng extends JavaImageConverters.BaseConverter { + public TIToPng() { + super(TestImageObject.class, Png.class); + } + + @Override + protected T doConversion(Image from, Class to) { + TestImageObject o = (TestImageObject) from.get(); + return (T) o.png; + } + } + + public static class PngToTI extends JavaImageConverters.BaseConverter { + public PngToTI() { + super(Png.class, TestImageObject.class); + } + + @Override + protected T doConversion(Image from, Class to) { + Png o = (Png) from.get(); + return (T) new TestImageObject(o); + } + } + + +} diff --git a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/JavaFormatsTest.java b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/JavaFormatsTest.java index 29232380e..22d12b8d0 100644 --- a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/JavaFormatsTest.java +++ b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/JavaFormatsTest.java @@ -20,12 +20,10 @@ import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.data.NDArray; -import ai.konduit.serving.pipeline.impl.format.SerializedNDArray; -import org.junit.Ignore; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; import org.junit.Test; import java.util.Arrays; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; diff --git a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/NDArrayTests.java b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/NDArrayTests.java new file mode 100644 index 000000000..b7dd2bb69 --- /dev/null +++ b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/data/NDArrayTests.java @@ -0,0 +1,157 @@ +/* + * ****************************************************************************** + * * Copyright (c) 2020 Konduit K.K. + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package ai.konduit.serving.pipeline.impl.data; + +import ai.konduit.serving.pipeline.api.data.NDArray; +import ai.konduit.serving.pipeline.api.data.NDArrayType; +import ai.konduit.serving.pipeline.api.format.NDArrayConverter; +import ai.konduit.serving.pipeline.api.format.NDArrayFactory; +import ai.konduit.serving.pipeline.api.format.NDArrayFormat; +import ai.konduit.serving.pipeline.impl.data.ndarray.BaseNDArray; +import ai.konduit.serving.pipeline.impl.data.ndarray.SerializedNDArray; +import ai.konduit.serving.pipeline.registry.NDArrayConverterRegistry; +import ai.konduit.serving.pipeline.registry.NDArrayFactoryRegistry; +import lombok.AllArgsConstructor; +import org.junit.Test; +import org.nd4j.common.base.Preconditions; + +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.util.Collections; +import java.util.Set; + +import static org.junit.Assert.assertArrayEquals; + +public class NDArrayTests { + + @Test + public void test2StepConversion(){ + //The idea: We add some new module with a new NDArray format, X + //In that module, we implement only conversion of X->SerializedNDArray and SerializedNDArray->X + //What if we want to do X->Y? + //NDArrayConverterRegistry will try to do X->SerializedNDArray->Y - as SerializedNDArray is something all image types should implement + // conversions to/from for + + float[] f = new float[]{1,2,3}; + + + NDArrayFactoryRegistry.addFactory(new TestNDArrayFactory()); + NDArrayConverterRegistry.addConverter(new TNDToSerializedNDArray()); + NDArrayConverterRegistry.addConverter(new SerializedNDArrayToTND()); + NDArray nd = NDArray.create(new TestNDArrayObject(f)); + + //TestNDArray -> SerializedNDArray -> float[] + float[] fArr = nd.getAs(float[].class); + assertArrayEquals(f, fArr, 0.0f); + + //float[] -> SerializedNDArray -> TestNDArray + NDArray i2 = NDArray.create(f); + TestNDArrayObject out = i2.getAs(TestNDArrayObject.class); + float[] outF = out.getF(); + assertArrayEquals(f, outF, 0.0f); + } + + @AllArgsConstructor + @lombok.Data + public static class TestNDArrayObject { + private float[] f; + } + + public static class TestNDArray extends BaseNDArray { + public TestNDArray(TestNDArrayObject o) { + super(o); + } + } + + public static class TestNDArrayFactory implements NDArrayFactory { + + @Override + public Set> supportedTypes() { + return Collections.singleton(TestNDArrayObject.class); + } + + @Override + public boolean canCreateFrom(Object o) { + return o instanceof TestNDArrayObject; + } + + @Override + public NDArray create(Object o) { + Preconditions.checkState(canCreateFrom(o)); + return new NDArrayTests.TestNDArray((TestNDArrayObject)o); + } + } + + public static class TNDToSerializedNDArray implements NDArrayConverter { + @Override + public boolean canConvert(NDArray from, NDArrayFormat to) { + return false; + } + + @Override + public boolean canConvert(NDArray from, Class to) { + return from.get().getClass() == TestNDArrayObject.class && to == SerializedNDArray.class; + } + + @Override + public T convert(NDArray from, NDArrayFormat to) { + throw new UnsupportedOperationException(); + } + + @Override + public T convert(NDArray from, Class to) { + Preconditions.checkState(canConvert(from, to)); + TestNDArrayObject arr = (TestNDArrayObject) from.get(); + float[] fArr = arr.getF(); + ByteBuffer bb = ByteBuffer.allocateDirect(fArr.length * 4); + bb.asFloatBuffer().put(fArr); + return (T) new SerializedNDArray(NDArrayType.FLOAT, new long[]{fArr.length}, bb); + } + } + + public static class SerializedNDArrayToTND implements NDArrayConverter { + + @Override + public boolean canConvert(NDArray from, NDArrayFormat to) { + return false; + } + + @Override + public boolean canConvert(NDArray from, Class to) { + return to == TestNDArrayObject.class && from.get().getClass() == SerializedNDArray.class; + } + + @Override + public T convert(NDArray from, NDArrayFormat to) { + throw new UnsupportedOperationException(); + } + + @Override + public T convert(NDArray from, Class to) { + Preconditions.checkState(canConvert(from, to)); + SerializedNDArray s = (SerializedNDArray)from.get(); + FloatBuffer fb = s.getBuffer().asFloatBuffer(); + int len = fb.capacity(); + float[] out = new float[len]; + fb.get(out); + return (T) new TestNDArrayObject(out); + } + } + +} diff --git a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/step/TestPipelineSteps.java b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/step/TestPipelineSteps.java index 686958f41..d815981b5 100644 --- a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/step/TestPipelineSteps.java +++ b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/step/TestPipelineSteps.java @@ -46,10 +46,10 @@ public void testLoggingPipeline(){ Data d = Data.singleton("someDouble", 1.0); d.put("someKey", "someValue"); - pe.exec(d); + pe.exec(null, d); assertEquals(1, count1.get()); assertEquals(1, count2.get()); - pe.exec(d); + pe.exec(null, d); assertEquals(2, count1.get()); assertEquals(2, count2.get()); } diff --git a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/util/CallbackPipelineStep.java b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/util/CallbackPipelineStep.java index c8801d19f..e71343e23 100644 --- a/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/util/CallbackPipelineStep.java +++ b/konduit-serving-pipeline/src/test/java/ai/konduit/serving/pipeline/impl/util/CallbackPipelineStep.java @@ -15,6 +15,7 @@ ******************************************************************************/ package ai.konduit.serving.pipeline.impl.util; +import ai.konduit.serving.pipeline.api.context.Context; import ai.konduit.serving.pipeline.api.data.Data; import ai.konduit.serving.pipeline.api.step.PipelineStep; import ai.konduit.serving.pipeline.api.step.PipelineStepRunner; @@ -63,7 +64,7 @@ public PipelineStep getPipelineStep() { } @Override - public Data exec(Data data) { + public Data exec(Context ctx, Data data) { step.consumer.accept(data); return data; } diff --git a/konduit-serving-pipeline/src/test/resources/data/5_32x32.png b/konduit-serving-pipeline/src/test/resources/data/5_32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..a21967b8acfbf15dd6a7980e08f22f7fd9681827 GIT binary patch literal 1095 zcmV-N1i1T&P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf1KdeOK~z{r?N`|= zbx{;w*EJNG$vlV11CmmvN2L&vJSgSOlRrRx{{c^i;(_ucuS%H;AtfT2XBi`z$MvnX zoO@lzaqe+Hbzi<;{o3d3vxnbad#$ziF#!M)or0vKBydrloreDcoG##>oX})40pff2 z^z?M_@$mtFe}6bWK8B^GB`7N^dv`P!7nq)&hMAceI6ps^_6+`JNB|GO6M1`k%LHCt zUc#uTD0q2!fyc*3nJhRs7(zosVR3O01_uYBxw#o)V`F8EmX;Q1YionGwKaKX*cHGl zbai!w%F0TahzCncOM|$$I0y?1gM@?xrR4{Xj*ehqVF9kMuOTNVN35~Fz7DOetuQ}7 z4>vbAN=pL*JUl!gA|e7dH#e1<&v<)#6S9RETy}PLLVth1h`+zT7eTkTw{UQ9AZ={H z0vIenlQf`|udlDP10S$`r&Zzb@UZfR*wE1MWt{Iw0?!r_4UES z!viEICrinDosp3NJv}|p(a|A(F*i2{tE;Qh&mIB;1Eu_;Oif>3UrbC)#DIVRtg5QQ zt*tHG-rmOA+FENHv)PQ%(b3Y@wyz%90*3Oy)zy{w!NtV|oSvSFgZcUSL27EM(gJ|C z1@`v#r0q{&I1#b4F_xE?CB%4p>gwvOW^@#~KA;8m1zu%HNC;$SXIsOHOfMF)2sHx3 zKAL5?~>vXPlUrP$VEy7DVz5YYx%luLxkWh~na6+}YU?5r>C|*xuf*+18f7 zCcr{E@k9V)e}5k-Of}nCXgCk(fjzmpx*8fA8-Ws$)ZN|P!S3#^gseR{bRXb@CNeTo zPP@FkJjlw*f}o%vh>3{-H#av)R6|2UP+nfH^w0)bK*}$s(IfA7K|ujFH8tVnLqmrWZ-F5TP@b2gnQuM3;);q2$jr=?dIl#aPQ$aavoJ6)0F)Qt z?Ceaw>A|iAIF!h=n39r$MMXtOgX8n_v$9@9x(Oj=LL40(#hRKL%`x<7Hwx(1gJWZ3 zczJoLO!)y8GBq`Y`T6;peT`6408h$^WPE%a$(xnwK7%ftSYKcNsh=a%6i`x9BCECq z?I-s3_Da6ziNnLgB^9gt{*;?SAO$;pW!2Qvbm$Dus%TVMfB z{WB5^bBJ;FQiDIL>-pb+{}a^suAY(