From 05a771a4c50c7d6531592d37b2892ab531b9878f Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Tue, 21 Nov 2023 09:30:55 -0700 Subject: [PATCH 1/7] Add initial standalone segmentation defs Signed-off-by: Alex-Brooks --- .../data_model/image_segmentation.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 caikit_computer_vision/data_model/image_segmentation.py diff --git a/caikit_computer_vision/data_model/image_segmentation.py b/caikit_computer_vision/data_model/image_segmentation.py new file mode 100644 index 0000000..f9f1e41 --- /dev/null +++ b/caikit_computer_vision/data_model/image_segmentation.py @@ -0,0 +1,44 @@ +# Copyright The Caikit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Data structures for segmentation in images.""" + + +# Standard +from typing import List + +# Third Party +from py_to_proto.dataclass_to_proto import Annotated, FieldNumber + +# First Party +from caikit.core import DataObjectBase, dataobject +from caikit.interfaces.vision import data_model as caikit_dm +from caikit.interfaces.common.data_model import ProducerId +import alog + +log = alog.use_channel("DATAM") + +@dataobject(package="caikit_data_model.caikit_computer_vision") +class ObjectSegment(DataObjectBase): + score: Annotated[float, FieldNumber(1)] + label: Annotated[str, FieldNumber(2)] + # TODO: We should be able to specify subtype, i.e., PIL image mode. + # This mask should be image mode (L), 8 bit grayscale image treated + # as a binary image, where 0 is background, and 255 is part of the + # object to align with HF task definitions. + mask: Annotated[caikit_dm.Image, FieldNumber(3)] + +@dataobject(package="caikit_data_model.caikit_computer_vision") +class ImageSegmentationResult(DataObjectBase): + object_segments: Annotated[List[ObjectSegment], FieldNumber(1)] + producer_id: Annotated[ProducerId, FieldNumber(2)] From 17b25dfc9aa0d4e6c2e840bce5300f356f092238 Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Tue, 21 Nov 2023 09:31:12 -0700 Subject: [PATCH 2/7] Add segmentation task def Signed-off-by: Alex-Brooks --- caikit_computer_vision/data_model/tasks.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/caikit_computer_vision/data_model/tasks.py b/caikit_computer_vision/data_model/tasks.py index d1236a1..e95697a 100644 --- a/caikit_computer_vision/data_model/tasks.py +++ b/caikit_computer_vision/data_model/tasks.py @@ -23,6 +23,7 @@ # Local from .image_classification import ImageClassificationResult +from .image_segmentation import ImageSegmentationResult from .object_detection import ObjectDetectionResult @@ -47,3 +48,15 @@ class ImageClassificationTask(TaskBase): and producing an iterable of objects containing class names and typically confidence scores. """ + + +@task( + required_parameters={"inputs": bytes}, + output_type=ImageSegmentationResult, +) +class ImageSegmentationTask(TaskBase): + """The image classification task is responsible for taking an input image + and producing a pixel mask with optional class names and confidence scores. + Note that at the moment, this task encapsulates all segmentation types, + I.e., instance, object, semantic, etc... + """ From cc50093f762c1d8f0f839a162825b4eda8243970 Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Tue, 21 Nov 2023 09:31:31 -0700 Subject: [PATCH 3/7] Add test for segmentation task Signed-off-by: Alex-Brooks --- tests/data_model/test_tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data_model/test_tasks.py b/tests/data_model/test_tasks.py index e641249..db758d0 100644 --- a/tests/data_model/test_tasks.py +++ b/tests/data_model/test_tasks.py @@ -64,6 +64,7 @@ class InvalidType: ( tasks.ObjectDetectionTask, tasks.ImageClassificationTask, + tasks.ImageSegmentationTask, ), ) def test_tasks( From 350ca454184bf332d162044190651dc56698c85b Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Tue, 21 Nov 2023 09:42:01 -0700 Subject: [PATCH 4/7] Formatting, add image seg defs to package init Signed-off-by: Alex-Brooks --- caikit_computer_vision/data_model/__init__.py | 3 ++- caikit_computer_vision/data_model/image_segmentation.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/caikit_computer_vision/data_model/__init__.py b/caikit_computer_vision/data_model/__init__.py index ac4effe..29a0d6f 100644 --- a/caikit_computer_vision/data_model/__init__.py +++ b/caikit_computer_vision/data_model/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. # Local -from . import image_classification, object_detection, tasks +from . import image_classification, image_segmentation, object_detection, tasks from .image_classification import * +from .image_segmentation import * from .object_detection import * diff --git a/caikit_computer_vision/data_model/image_segmentation.py b/caikit_computer_vision/data_model/image_segmentation.py index f9f1e41..1b76136 100644 --- a/caikit_computer_vision/data_model/image_segmentation.py +++ b/caikit_computer_vision/data_model/image_segmentation.py @@ -22,12 +22,13 @@ # First Party from caikit.core import DataObjectBase, dataobject -from caikit.interfaces.vision import data_model as caikit_dm from caikit.interfaces.common.data_model import ProducerId +from caikit.interfaces.vision import data_model as caikit_dm import alog log = alog.use_channel("DATAM") + @dataobject(package="caikit_data_model.caikit_computer_vision") class ObjectSegment(DataObjectBase): score: Annotated[float, FieldNumber(1)] @@ -38,6 +39,7 @@ class ObjectSegment(DataObjectBase): # object to align with HF task definitions. mask: Annotated[caikit_dm.Image, FieldNumber(3)] + @dataobject(package="caikit_data_model.caikit_computer_vision") class ImageSegmentationResult(DataObjectBase): object_segments: Annotated[List[ObjectSegment], FieldNumber(1)] From 85f5d66f7c34ce05c37554a2a006a3ef900ece38 Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Tue, 21 Nov 2023 10:00:04 -0700 Subject: [PATCH 5/7] Add segmentation info to object detection Signed-off-by: Alex-Brooks --- .../data_model/object_detection.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/caikit_computer_vision/data_model/object_detection.py b/caikit_computer_vision/data_model/object_detection.py index 088ed70..5ecae2f 100644 --- a/caikit_computer_vision/data_model/object_detection.py +++ b/caikit_computer_vision/data_model/object_detection.py @@ -24,9 +24,18 @@ from caikit.interfaces.common.data_model import ProducerId import alog +# Local +from .image_segmentation import ObjectSegment + log = alog.use_channel("DATAM") +@dataobject(package="caikit_data_model.caikit_computer_vision") +class Point2f(DataObjectBase): + x: Annotated[float, FieldNumber(1)] + y: Annotated[float, FieldNumber(2)] + + @dataobject(package="caikit_data_model.caikit_computer_vision") class BoundingBox(DataObjectBase): xmin: Annotated[int, FieldNumber(1)] @@ -40,6 +49,11 @@ class DetectedObject(DataObjectBase): score: Annotated[float, FieldNumber(1)] label: Annotated[str, FieldNumber(2)] box: Annotated[BoundingBox, FieldNumber(3)] + # Optional segmentation information, containing a list of pixel + # coordinates representing the segmentation mask of the object. + object_segments: Annotated[List[Point2f], FieldNumber(4)] + # Optional run-length encoding of the object being described. + rle: Annotated[str, FieldNumber(5)] @dataobject(package="caikit_data_model.caikit_computer_vision") From 607fd35dc49a78ec2c082c586e7e8df3a5d985d6 Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Tue, 21 Nov 2023 10:11:25 -0700 Subject: [PATCH 6/7] Add tentative anomaly detection info for object detection Signed-off-by: Alex-Brooks --- .../data_model/object_detection.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/caikit_computer_vision/data_model/object_detection.py b/caikit_computer_vision/data_model/object_detection.py index 5ecae2f..ad3a534 100644 --- a/caikit_computer_vision/data_model/object_detection.py +++ b/caikit_computer_vision/data_model/object_detection.py @@ -29,13 +29,19 @@ log = alog.use_channel("DATAM") - +# Image coordinates - TODO: Probably should standardize what we use for these... @dataobject(package="caikit_data_model.caikit_computer_vision") class Point2f(DataObjectBase): x: Annotated[float, FieldNumber(1)] y: Annotated[float, FieldNumber(2)] +@dataobject(package="caikit_data_model.caikit_computer_vision") +class Point2d(DataObjectBase): + x: Annotated[int, FieldNumber(1)] + y: Annotated[int, FieldNumber(2)] + + @dataobject(package="caikit_data_model.caikit_computer_vision") class BoundingBox(DataObjectBase): xmin: Annotated[int, FieldNumber(1)] @@ -44,16 +50,35 @@ class BoundingBox(DataObjectBase): ymax: Annotated[int, FieldNumber(4)] +@dataobject(package="caikit_data_model.caikit_computer_vision") +class AnomalyRegion(DataObjectBase): + score: Annotated[float, FieldNumber(1)] + # Bounding box and primary focus area of the detected anomaly; + # note that these coordinates are relative to the detected object. + box: Annotated[BoundingBox, FieldNumber(2)] + anomaly_hotspot: Annotated[Point2d, FieldNumber(3)] + + +@dataobject(package="caikit_data_model.caikit_computer_vision") +class Anomaly(DataObjectBase): + score: Annotated[float, FieldNumber(1)] + anomaly_threshold: Annotated[float, FieldNumber(2)] + detail_data: Annotated[str, FieldNumber(3)] + regions: Annotated[List[AnomalyRegion], FieldNumber(4)] + + @dataobject(package="caikit_data_model.caikit_computer_vision") class DetectedObject(DataObjectBase): score: Annotated[float, FieldNumber(1)] label: Annotated[str, FieldNumber(2)] box: Annotated[BoundingBox, FieldNumber(3)] - # Optional segmentation information, containing a list of pixel - # coordinates representing the segmentation mask of the object. + ### Optional segmentation information + # list of pixel coordinates representing the segmentation mask of the object. object_segments: Annotated[List[Point2f], FieldNumber(4)] # Optional run-length encoding of the object being described. rle: Annotated[str, FieldNumber(5)] + ### Optional anomaly detection information + anomaly: Annotated[Anomaly, FieldNumber(6)] @dataobject(package="caikit_data_model.caikit_computer_vision") From 6cc315fa898e54102750725338b86b6f811dfa98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Dec 2023 19:54:06 +0000 Subject: [PATCH 7/7] Update caikit[interfaces-vision,runtime-grpc,runtime-http] requirement Updates the requirements on [caikit[interfaces-vision,runtime-grpc,runtime-http]](https://github.com/caikit/caikit) to permit the latest version. - [Release notes](https://github.com/caikit/caikit/releases) - [Changelog](https://github.com/caikit/caikit/blob/main/releases.md) - [Commits](https://github.com/caikit/caikit/compare/v0.16.0...v0.26.0) --- updated-dependencies: - dependency-name: caikit[interfaces-vision,runtime-grpc,runtime-http] dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2e21f1f..250d541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers=[ "License :: OSI Approved :: Apache Software License" ] dependencies = [ - "caikit[runtime-grpc,runtime-http,interfaces-vision]>=0.16.0,<0.24.0", + "caikit[runtime-grpc,runtime-http,interfaces-vision]>=0.16.0,<0.27.0", "transformers>=4.27.1,<5", "torch>2.0,<3", "timm>=0.9.5,<1"