From 662c5cb66fca44905edeef3de30e32fe08d1b8a0 Mon Sep 17 00:00:00 2001 From: Ryan Bannon Date: Wed, 11 Jun 2014 12:19:01 -0400 Subject: [PATCH 1/3] Squashed commit of the following: commit 2bd0c09dea71734c7bc2b20b0d6e0c1134e56cc2 Author: Rodan Date: Wed Jun 11 12:15:19 2014 -0400 Fix: memory issue for lyric extraction correction post-process Had to process the final page in chunks, not all at once. commit 97861563f979bc8c03f24c7552ecfdd085d73320 Author: Ryan Bannon Date: Wed Jun 11 11:25:04 2014 -0400 Fix: forgot "self." because I'm stupid commit 59ecb0bab035735b15bcf42d16a850b663522817 Author: Ryan Bannon Date: Wed Jun 11 11:22:44 2014 -0400 Fix: function sig wrong commit 5980b294a1d619443c28937f3a1812e37f0f2ac2 Author: Ryan Bannon Date: Wed Jun 11 11:19:27 2014 -0400 Fix: syntax (I'm an idiot) commit 4384eb7dc7ba5e073501021795d6a744b0c22193 Author: Ryan Bannon Date: Wed Jun 11 11:16:57 2014 -0400 Fix: syntax (I hate Python) commit 56b477df20e419e2303aee77a09766a72e177e3a Author: Ryan Bannon Date: Wed Jun 11 11:15:35 2014 -0400 Fix: syntax error commit 83b597c51655be0a44ff0485930046d4e771cf7c Author: Ryan Bannon Date: Wed Jun 11 11:12:28 2014 -0400 Fix attempt: memory error Fixing Memory Error when processing lyric extraction correction post-processing. --- .../custom/pixel_segment/celery_task.py | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/rodan/jobs/gamera/custom/pixel_segment/celery_task.py b/rodan/jobs/gamera/custom/pixel_segment/celery_task.py index 8f271c9c3..28af889fa 100644 --- a/rodan/jobs/gamera/custom/pixel_segment/celery_task.py +++ b/rodan/jobs/gamera/custom/pixel_segment/celery_task.py @@ -6,6 +6,9 @@ from rodan.jobs.gamera.custom.gamera_custom_base import GameraCustomTask class PixelSegmentTask(GameraCustomTask): + + COLOUR_SWAP_PIXELS_BOX_HEIGHT = 100 + max_retries = None name = 'gamera.custom.lyric_extraction.pixel_segment' @@ -41,8 +44,7 @@ def process_image(self, task_image, settings): # Convert red to white and black to green. colour_swap = {(255, 0, 0): (255, 255, 255), (0, 255, 0): (0, 0, 0)} - box = (0, 0, imageOriginal.size[0], imageOriginal.size[1]) - imageOriginal = self._colour_swap_pixels(imageOriginal, box, colour_swap) + imageOriginal = self._colour_swap_pixels(imageOriginal, colour_swap) # Convert back to gamera image and return. finalImage = from_pil(imageOriginal) @@ -136,15 +138,31 @@ def _colour_pixels(self, image, box, colour, collisionGeometry): # Colours all pixels defined in "swap" keys with their "swap" values for the box dimensions. # See other '_colour_pixels_' methods...very similar. - def _colour_swap_pixels(self, image, box, colour_swap): - imageCrop = image.crop(box) - imageCrop.load() - imagePixelData = list(imageCrop.getdata()) - for i in range(len(imagePixelData)): - if (imagePixelData[i] in colour_swap): - imagePixelData[i] = colour_swap[imagePixelData[i]] - imageCrop.putdata(imagePixelData) - image.paste(imageCrop, box) + # + # NOTE: we're not doing the swap on the entire image at once as this can cause memory proglems + # on large images. Rather, we do vertical sections of the image one at a time. + def _colour_swap_pixels(self, image, colour_swap): + imageHeight = image.size[1] + count = 0 + done = False + while not done: + upperLeftY = count * self.COLOUR_SWAP_PIXELS_BOX_HEIGHT + lowerRightY = upperLeftY + self.COLOUR_SWAP_PIXELS_BOX_HEIGHT + if lowerRightY >= imageHeight: + lowerRightY = imageHeight + done = True + box = (0, upperLeftY, image.size[0], lowerRightY) + imageCrop = image.crop(box) + imageCrop.load() + imagePixelData = list(imageCrop.getdata()) + for i in range(len(imagePixelData)): + if (imagePixelData[i] in colour_swap): + imagePixelData[i] = colour_swap[imagePixelData[i]] + imageCrop.putdata(imagePixelData) + image.paste(imageCrop, box) + del imagePixelData + del imageCrop + count += 1 return image From f54cd2ebfbdf6339dc5901323d210b56716a545e Mon Sep 17 00:00:00 2001 From: Ryan Bannon Date: Mon, 30 Jun 2014 13:16:30 -0400 Subject: [PATCH 2/3] Feature: added support for CORS --- requirements.txt | 1 + rodan/settings_production.py.example | 17 +++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index d031c4c30..26502cc98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ anyjson==0.3.3 backports.ssl-match-hostname==3.4.0.2 billiard==3.3.0.17 celery==3.1.11 +django-cors-headers django-extensions==1.3.5 django-filter==0.7 django-guardian==1.2.0 diff --git a/rodan/settings_production.py.example b/rodan/settings_production.py.example index d43c3a311..f889ecae7 100644 --- a/rodan/settings_production.py.example +++ b/rodan/settings_production.py.example @@ -38,22 +38,19 @@ MEDIA_URL = "/uploads/" # When you find a better solution for returning the absolute url from a celery job, # go clean up the code in these places. BASE_URL = 'http://localhost:8000' +SOLR_URL = 'http://rodan.simssa.ca:8080/rodan-search-dev' +IIP_URL = 'http://rodan.simssa.ca/fcgi-bin/iipsrv.fcgi' +############################################################################### +## Celery configuration +############################################################################### +BROKER_CONNECTION_MAX_RETRIES = '0' BROKER_URL = 'amqp://rodanuser:DDMALrodan@localhost:5672/DDMAL' - +CELERY_ENABLE_UTC = True CELERY_IMPORTS = ("rodan.helpers.thumbnails", "rodan.helpers.convert", "rodan.helpers.processed", "rodan.helpers.resultspackagemanager", "rodan.jobs",) - -# TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner' - CELERY_RESULT_BACKEND = "amqp" -#Note: If youre using SQLite as the Django database backend, celeryd will only be able to process one task at a time, -#this is because SQLite doesnt allow concurrent writes. -# CELERY_RESULT_DBURI = "sqlite:///db.sqlite" - -SOLR_URL = 'http://rodan.simssa.ca:8080/rodan-search-dev' -IIP_URL = 'http://rodan.simssa.ca/fcgi-bin/iipsrv.fcgi' From d3ce985d1a41f329298e2b1829bd129391a7b244 Mon Sep 17 00:00:00 2001 From: Ryan Bannon Date: Mon, 30 Jun 2014 13:25:38 -0400 Subject: [PATCH 3/3] Fix: CORS settings for django --- rodan/settings_production.py.example | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/rodan/settings_production.py.example b/rodan/settings_production.py.example index f889ecae7..820ff88fb 100644 --- a/rodan/settings_production.py.example +++ b/rodan/settings_production.py.example @@ -54,3 +54,16 @@ CELERY_IMPORTS = ("rodan.helpers.thumbnails", "rodan.jobs",) CELERY_RESULT_BACKEND = "amqp" +############################################################################### +## Django CORS configuration +############################################################################### +#CORS_ORIGIN_ALLOW_ALL = False +#CORS_ORIGIN_WHITELIST = ( +# 'some domain or IP' +#) +#CORS_ALLOW_CREDENTIALS = True +#CORS_EXPOSE_HEADERS = ( +# 'Set-Cookie', +# 'Vary' +#) +