From 71848f9676192b9ca45494f9c7cfc723beec347e Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:38:57 -0400 Subject: [PATCH 1/6] add rename method for Resource --- rodan-main/code/rodan/models/resource.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rodan-main/code/rodan/models/resource.py b/rodan-main/code/rodan/models/resource.py index 422cf1547..e710f72ed 100644 --- a/rodan-main/code/rodan/models/resource.py +++ b/rodan-main/code/rodan/models/resource.py @@ -142,6 +142,10 @@ def __str__(self): labels = models.ManyToManyField(ResourceLabel, blank=True) + def rename(self, newname): + self.name = newname + self.save() + def save(self, *args, **kwargs): super(Resource, self).save(*args, **kwargs) if not os.path.exists(self.resource_path): From 5cb1814bfb4144bb81b8376900ff4ae2cc0b82cb Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:53:37 -0400 Subject: [PATCH 2/6] allow user to rename a file in resource distributor the test method has not been fixed yet --- .../code/rodan/jobs/resource_distributor.py | 108 +++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/rodan-main/code/rodan/jobs/resource_distributor.py b/rodan-main/code/rodan/jobs/resource_distributor.py index ea0095837..df3762815 100644 --- a/rodan-main/code/rodan/jobs/resource_distributor.py +++ b/rodan-main/code/rodan/jobs/resource_distributor.py @@ -2,7 +2,24 @@ import shutil from rodan.jobs.base import RodanTask -from rodan.models import ResourceType +from rodan.models import ResourceType, Input, Output +from django.conf import settings as rodan_settings + +import logging + +logger = logging.getLogger("rodan") + +def log_structure(data, level=0): + if isinstance(data, dict): + for key, value in data.items(): + logger.info("%sKey: %s", " " * level, key) + log_structure(value, level + 1) + elif isinstance(data, list): + for idx, item in enumerate(data): + logger.info("%sIndex %d:", " " * level, idx) + log_structure(item, level + 1) + else: + logger.info("%sValue: %s", " " * level, data) class ResourceDistributor(RodanTask): @@ -18,6 +35,16 @@ class ResourceDistributor(RodanTask): 'type': 'string', 'default': 'application/octet-stream', 'description': 'Specifies the eligible resource types for input' + }, + 'User custom prefix': { + 'type': 'string', + 'default': 'custom prefix - ', + 'description': 'User specified prefix (please also include space, hyphen, etc.)' + }, + 'User custom suffix': { + 'type': 'string', + 'default': '- custom suffix', + 'description': 'User specified suffix (please also include space, hyphen, etc.)' } } } @@ -44,8 +71,66 @@ class ResourceDistributor(RodanTask): }, ) + + def _inputs(self, runjob, with_urls=False): + """ + Return a dictionary of list of input file path and input resource type. + If with_urls=True, it also includes the resource url and thumbnail urls. + """ + + self.runjob = runjob + + def _extract_resource(resource, resource_type_mimetype=None): + r = { + # convert 'unicode' object to 'str' object for consistency + "resource_path": str(resource.resource_file.path), + "resource_type": str( + resource_type_mimetype or resource.resource_type.mimetype + ), + "resource": resource, + } + if with_urls: + r["resource_url"] = str(resource.resource_url) + r["diva_object_data"] = str(resource.diva_json_url) + r["diva_iip_server"] = getattr(rodan_settings, "IIPSRV_URL") + r["diva_image_dir"] = str(resource.diva_image_dir) + return r + + input_objs = ( + Input.objects.filter(run_job=runjob) + .select_related("resource", "resource__resource_type", "resource_list") + .prefetch_related("resource_list__resources") + ) + + inputs = {} + for input in input_objs: + ipt_name = str(input.input_port_type_name) + if ipt_name not in inputs: + inputs[ipt_name] = [] + if input.resource is not None: # If resource + inputs[ipt_name].append(_extract_resource(input.resource)) + elif input.resource_list is not None: # If resource_list + inputs[ipt_name].append( + map( + lambda x: _extract_resource( + x, input.resource_list.get_resource_type().mimetype + ), + input.resource_list.resources.all(), + ) + ) + else: + raise RuntimeError( + ( + "Cannot find any resource or resource list on Input" " {0}" + ).format(input.uuid) + ) + return inputs + def run_my_task(self, inputs, settings, outputs): + # log_structure(inputs) input_type = inputs['Resource input'][0]['resource_type'] + input_resource = inputs['Resource input'][0]['resource'] + input_name = input_resource.name valid_input_type_num = settings['Resource type'] valid_input_type = self.settings['properties']['Resource type']['enum'][valid_input_type_num] # noqa if input_type != valid_input_type: @@ -57,8 +142,29 @@ def run_my_task(self, inputs, settings, outputs): ).format(input_type, valid_input_type) ) return False + prefix = settings["User custom prefix"] + if not isinstance(prefix, str): + self.my_error_information( + None, + ("User custom prefix can only be strings") + ) + return False + suffix = settings["User custom suffix"] + if not isinstance(suffix, str): + self.my_error_information( + None, + ("User custom suffix can only be strings") + ) + return False + new_name = prefix + input_name + suffix + assert isinstance(new_name,str) + + input_resource.rename(new_name) + input_resource.save(update_fields=["resource_file"]) + outfile_path = outputs['Resource output'][0]['resource_path'] infile_path = inputs['Resource input'][0]['resource_path'] + shutil.copyfile(infile_path, outfile_path) return True From e0a9c2a6238913f6008ade0cc4e94c495684cfa9 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:21:03 -0400 Subject: [PATCH 3/6] modify test for resource distributor need to be tested locally --- .../code/rodan/jobs/resource_distributor.py | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/rodan-main/code/rodan/jobs/resource_distributor.py b/rodan-main/code/rodan/jobs/resource_distributor.py index df3762815..cbfe6016c 100644 --- a/rodan-main/code/rodan/jobs/resource_distributor.py +++ b/rodan-main/code/rodan/jobs/resource_distributor.py @@ -176,27 +176,38 @@ def test_my_task(self, testcase): import PIL.Image import numpy as np resource_types_list = list(map(lambda rt: str(rt.mimetype), ResourceType.objects.all())) + from model_mommy import mommy + from rodan.models import Resource, ResourceType + + # Create a Resource instance using mommy + resource_type = mommy.make(ResourceType, mimetype="image/rgb+png") + rc = mommy.make(Resource, resource_type=resource_type) - # Not so sure what this job is for, but I'll use image/png as the testcase. inputs = { - "Resource input": [ - { - 'resource_type': 'image/rgb+png', - 'resource_path': testcase.new_available_path() + "Resource input": [ + { + "resource_path": rc.resource_path, + "resource_type": rc.resource_type.mimetype, + "resource": rc + } + ] } - ] - } outputs = { - "Resource output": [ - { - "resource_type": "image/rgb+png", - "resource_path": testcase.new_available_path() + "Resource output": [ + { + "resource_type": "image/rgb+png", + "resource_path": testcase.new_available_path() + } + ] } - ] - } settings = { - "Resource type": resource_types_list.index("image/rgb+png") - } + "Resource type": resource_types_list.index("image/rgb+png"), + "User custom prefix": "test prefix - ", + "User custom suffix": "- test suffix" + } + + original_image = rc.name + PIL.Image.new("RGB", size=(50, 50), color=(255, 0, 0)).save(inputs['Resource input'][0]['resource_path'], 'PNG') array_gt = np.zeros((50, 50, 3)).astype(np.uint8) array_gt[:, :, 0] = 255 @@ -211,3 +222,6 @@ def test_my_task(self, testcase): testcase.assertEqual(result.format, 'PNG') # and the data should be identical np.testing.assert_equal(array_gt, array_result) + + new_name = f"{settings['User custom prefix']}{original_image}{settings['User custom suffix']}" + testcase.assertEqual(inputs['Resource input'][0]['resource'].name, new_name) \ No newline at end of file From a3b4d4cacbed73646620ea036dcda551badf1c72 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Fri, 1 Nov 2024 13:24:21 -0400 Subject: [PATCH 4/6] fix unit test --- rodan-main/code/rodan/jobs/resource_distributor.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rodan-main/code/rodan/jobs/resource_distributor.py b/rodan-main/code/rodan/jobs/resource_distributor.py index cbfe6016c..8ff69773c 100644 --- a/rodan-main/code/rodan/jobs/resource_distributor.py +++ b/rodan-main/code/rodan/jobs/resource_distributor.py @@ -175,18 +175,18 @@ def my_error_information(self, exc, traceback): def test_my_task(self, testcase): import PIL.Image import numpy as np - resource_types_list = list(map(lambda rt: str(rt.mimetype), ResourceType.objects.all())) - from model_mommy import mommy from rodan.models import Resource, ResourceType + resource_types_list = list(map(lambda rt: str(rt.mimetype), ResourceType.objects.all())) + from model_mommy import mommy # Create a Resource instance using mommy - resource_type = mommy.make(ResourceType, mimetype="image/rgb+png") - rc = mommy.make(Resource, resource_type=resource_type) + resource_type, created = ResourceType.objects.get_or_create(mimetype="image/rgb+png") + rc = mommy.make(Resource, resource_type=resource_type, name="test_filename") inputs = { "Resource input": [ { - "resource_path": rc.resource_path, + "resource_path": testcase.new_available_path(), "resource_type": rc.resource_type.mimetype, "resource": rc } @@ -223,5 +223,6 @@ def test_my_task(self, testcase): # and the data should be identical np.testing.assert_equal(array_gt, array_result) + # Test name change new_name = f"{settings['User custom prefix']}{original_image}{settings['User custom suffix']}" testcase.assertEqual(inputs['Resource input'][0]['resource'].name, new_name) \ No newline at end of file From 249fa6255aa348769358449c27498bf590e92997 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:40:11 -0500 Subject: [PATCH 5/6] change default app type to rgba+png This is the default type for layers from classifying job (where the rename feature is mostly needed) --- rodan-main/code/rodan/jobs/resource_distributor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rodan-main/code/rodan/jobs/resource_distributor.py b/rodan-main/code/rodan/jobs/resource_distributor.py index 8ff69773c..f1740b061 100644 --- a/rodan-main/code/rodan/jobs/resource_distributor.py +++ b/rodan-main/code/rodan/jobs/resource_distributor.py @@ -33,7 +33,7 @@ class ResourceDistributor(RodanTask): 'Resource type': { 'enum': list(map(lambda rt: str(rt.mimetype), ResourceType.objects.all())), 'type': 'string', - 'default': 'application/octet-stream', + 'default': 'image/rgba+png', 'description': 'Specifies the eligible resource types for input' }, 'User custom prefix': { From 981e8d5428dae577be28b6e8a1d5fc738eb84843 Mon Sep 17 00:00:00 2001 From: Hanwen Zh <70364944+homework36@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:17:26 -0500 Subject: [PATCH 6/6] modify err msg for mismatched input type so that the user will understand they need to set the type in the job accordingly --- rodan-main/code/rodan/jobs/resource_distributor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rodan-main/code/rodan/jobs/resource_distributor.py b/rodan-main/code/rodan/jobs/resource_distributor.py index f1740b061..7936b9eee 100644 --- a/rodan-main/code/rodan/jobs/resource_distributor.py +++ b/rodan-main/code/rodan/jobs/resource_distributor.py @@ -137,7 +137,7 @@ def run_my_task(self, inputs, settings, outputs): self.my_error_information( None, ( - "Input cannot be of type {0}. Valid input set in setting is " + "Mismatched input of type {0}. The input type in job setting is " "{1}" ).format(input_type, valid_input_type) )