From da71a6fe2263073e22782a108b36bc4b8b80b668 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Tue, 8 Feb 2022 11:32:21 -0500 Subject: [PATCH 1/5] black and flake 8 updates --- concordia/tasks.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/concordia/tasks.py b/concordia/tasks.py index b0bc2b58e..c2a56c9d8 100644 --- a/concordia/tasks.py +++ b/concordia/tasks.py @@ -353,6 +353,52 @@ def calculate_difficulty_values(asset_qs=None): return updated_count +# modify above def for storage_image populate +@celery_app.task +def populate_storage_image_values(asset_qs=None): + """ + For Assets that existed prior to implementing the storage_image ImageField, build + the relative S3 storage key for the asset and update the storage_image value + """ + + # only fetch assest with no storgae image value + asset_qs = ( + Asset.objects.filter(storage_image__isnull=True) + .order_by("title") + .prefetch_related("item") + ) + + updated_count = 0 + + # We'll process assets in chunks using an iterator to avoid saving objects + # which will never be used again in memory. We will build the S3 relative key for + # each existing asset and pass them to bulk_update() to be saved in a single query. + for asset_chunk in chunked(asset_qs.iterator(), 3000): + changed_assets = [] + + for asset in asset_chunk: + asset.storage_image = "/".join( + [ + asset.item.project.campaign.slug, + asset.item.project.slug, + asset.item.item_id, + asset.media_url, + ] + ) + changed_assets.append(asset) + + if changed_assets: + # We will only save the new storage image value both for performance + # and to avoid any possibility of race conditions causing stale data + # to be saved: + Asset.objects.bulk_update(changed_assets, ["storage_image"]) + updated_count += len(changed_assets) + + logger.debug("Storage image updated count %s" % updated_count) + + return updated_count + + @celery_app.task def populate_asset_years(): """ From 4537c2e17aa33489aa0806aca6c9e97c9f460125 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Wed, 9 Feb 2022 10:43:30 -0500 Subject: [PATCH 2/5] add ordering on qs --- concordia/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concordia/tasks.py b/concordia/tasks.py index c2a56c9d8..44cf76087 100644 --- a/concordia/tasks.py +++ b/concordia/tasks.py @@ -364,7 +364,7 @@ def populate_storage_image_values(asset_qs=None): # only fetch assest with no storgae image value asset_qs = ( Asset.objects.filter(storage_image__isnull=True) - .order_by("title") + .order_by("id") .prefetch_related("item") ) From 25df6931a3b89b95a258378530d8048e9878e830 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Wed, 23 Feb 2022 07:31:57 -0500 Subject: [PATCH 3/5] black changes --- concordia/tasks.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/concordia/tasks.py b/concordia/tasks.py index 44cf76087..2f86370d0 100644 --- a/concordia/tasks.py +++ b/concordia/tasks.py @@ -363,9 +363,9 @@ def populate_storage_image_values(asset_qs=None): # only fetch assest with no storgae image value asset_qs = ( - Asset.objects.filter(storage_image__isnull=True) - .order_by("id") - .prefetch_related("item") + Asset.objects.filter(storage_image__isnull=True).order_by("id") + # .prefetch_related("item__project__campaign")[:20000] + .select_related("item__project__campaign")[:25000] ) updated_count = 0 @@ -373,8 +373,7 @@ def populate_storage_image_values(asset_qs=None): # We'll process assets in chunks using an iterator to avoid saving objects # which will never be used again in memory. We will build the S3 relative key for # each existing asset and pass them to bulk_update() to be saved in a single query. - for asset_chunk in chunked(asset_qs.iterator(), 3000): - changed_assets = [] + for asset_chunk in chunked(asset_qs.iterator(), 2000): for asset in asset_chunk: asset.storage_image = "/".join( @@ -385,14 +384,13 @@ def populate_storage_image_values(asset_qs=None): asset.media_url, ] ) - changed_assets.append(asset) - if changed_assets: - # We will only save the new storage image value both for performance - # and to avoid any possibility of race conditions causing stale data - # to be saved: - Asset.objects.bulk_update(changed_assets, ["storage_image"]) - updated_count += len(changed_assets) + # We will only save the new storage image value both for performance + # and to avoid any possibility of race conditions causing stale data + # to be saved: + + Asset.objects.bulk_update(asset_chunk, ["storage_image"]) + updated_count += len(asset_chunk) logger.debug("Storage image updated count %s" % updated_count) From 2a7e717e6753893d1ebcb2ce59394e9ea5623291 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Wed, 23 Feb 2022 09:34:06 -0500 Subject: [PATCH 4/5] back changes --- concordia/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/concordia/tasks.py b/concordia/tasks.py index 2f86370d0..483ee5a83 100644 --- a/concordia/tasks.py +++ b/concordia/tasks.py @@ -363,9 +363,9 @@ def populate_storage_image_values(asset_qs=None): # only fetch assest with no storgae image value asset_qs = ( - Asset.objects.filter(storage_image__isnull=True).order_by("id") - # .prefetch_related("item__project__campaign")[:20000] - .select_related("item__project__campaign")[:25000] + Asset.objects.filter(storage_image__isnull=True) + .order_by("id") + .prefetch_related("item__project__campaign")[:20000] ) updated_count = 0 From c76f65f473ddd13af94f778050ccade87722dbe7 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Wed, 23 Feb 2022 09:47:24 -0500 Subject: [PATCH 5/5] fix bad conflict resolution change --- concordia/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/concordia/tasks.py b/concordia/tasks.py index 157dcc3dd..483ee5a83 100644 --- a/concordia/tasks.py +++ b/concordia/tasks.py @@ -363,7 +363,6 @@ def populate_storage_image_values(asset_qs=None): # only fetch assest with no storgae image value asset_qs = ( - 1593-populate-existing-asset-storage-image Asset.objects.filter(storage_image__isnull=True) .order_by("id") .prefetch_related("item__project__campaign")[:20000]