From 10c5bf70d6a49fd080285e96e453e7ee832eba70 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Thu, 2 Dec 2021 13:18:40 -0500 Subject: [PATCH 1/7] Example of how to do S3 delete for asset files --- concordia/admin/actions.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/concordia/admin/actions.py b/concordia/admin/actions.py index eb9394dfd..43d2d8dac 100644 --- a/concordia/admin/actions.py +++ b/concordia/admin/actions.py @@ -1,10 +1,15 @@ import uuid +from logging import getLogger +import boto3 +from django.conf import settings from django.contrib import messages from django.utils.timezone import now from ..models import Asset, Transcription, TranscriptionStatus +logger = getLogger(__name__) + def anonymize_action(modeladmin, request, queryset): count = queryset.count() @@ -21,6 +26,18 @@ def anonymize_action(modeladmin, request, queryset): anonymize_action.short_description = "Anonymize and disable user accounts" +def delete_asset_with_s3_delete(modeladmin, request, queryset): + s3_client = boto3.client("s3") + for asset in queryset: + response = s3_client.delete_object( + Bucket=settings.S3_BUCKET_NAME, Key=asset.get_storage_url() + ) + logger.info(response) + + +delete_asset_with_s3_delete.short_description = "Delete asset and image files" + + def publish_item_action(modeladmin, request, queryset): """ Mark all of the selected items and their related assets as published From c56ec37d70b3740119fb1b75b6c61b6581d2c5d8 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Thu, 2 Dec 2021 13:23:34 -0500 Subject: [PATCH 2/7] Example of how to do S3 delete for asset files --- concordia/admin/actions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/concordia/admin/actions.py b/concordia/admin/actions.py index 43d2d8dac..a3f3c3511 100644 --- a/concordia/admin/actions.py +++ b/concordia/admin/actions.py @@ -33,6 +33,7 @@ def delete_asset_with_s3_delete(modeladmin, request, queryset): Bucket=settings.S3_BUCKET_NAME, Key=asset.get_storage_url() ) logger.info(response) + asset.delete() delete_asset_with_s3_delete.short_description = "Delete asset and image files" From e5c466953ecbefb49301b25b31aecd4f081e31e4 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Tue, 7 Dec 2021 10:45:52 -0500 Subject: [PATCH 3/7] post black reformatting --- concordia/models.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/concordia/models.py b/concordia/models.py index 27c94bf32..c17e6f2b0 100644 --- a/concordia/models.py +++ b/concordia/models.py @@ -109,6 +109,10 @@ def __str__(self): def get_absolute_url(self): return reverse("transcriptions:campaign-detail", args=(self.slug,)) + def get_storage_url(self): + s3_target = "/" + self.slug + return s3_target + class Topic(models.Model): objects = UnlistedPublicationQuerySet.as_manager() @@ -187,6 +191,10 @@ def get_absolute_url(self): kwargs={"campaign_slug": self.campaign.slug, "slug": self.slug}, ) + def get_storage_url(self): + s3_target = "/" + self.campaign.slug + "/" + self.slug + return s3_target + class Item(MetricsModelMixin("item"), models.Model): objects = PublicationQuerySet.as_manager() @@ -226,6 +234,17 @@ def get_absolute_url(self): }, ) + def get_storage_url(self): + s3_target = ( + "/" + + self.project.campaign.slug + + "/" + + self.project.slug + + "/" + + self.item_id + ) + return s3_target + class AssetQuerySet(PublicationQuerySet): def add_contribution_counts(self): @@ -301,6 +320,27 @@ def get_absolute_url(self): def latest_transcription(self): return self.transcription_set.order_by("-pk").first() + def get_storage_url(self): + s3_target = ( + "/" + + self.item.project.campaign.slug + + "/" + + self.item.project.slug + + "/" + + self.item.item_id + + "/" + + self.media_url + ) + return s3_target + + def get_storage_urlv2(self): + return "/".join( + self.item.project.campaign.slug, + self.item.project.slug, + self.item.item_id, + self.media_url, + ) + class Tag(MetricsModelMixin("tag"), models.Model): TAG_VALIDATOR = RegexValidator(r"^[- _À-ž'\w]{1,50}$") From 7a7052e363a20d14c26e6b7bbb5363b32a95bba9 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Tue, 14 Dec 2021 15:39:43 -0500 Subject: [PATCH 4/7] added count message + post black flake8 --- concordia/admin/__init__.py | 2 ++ concordia/admin/actions.py | 13 +++++++++++-- concordia/models.py | 37 +++---------------------------------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/concordia/admin/__init__.py b/concordia/admin/__init__.py index 089b7d199..282716e68 100644 --- a/concordia/admin/__init__.py +++ b/concordia/admin/__init__.py @@ -40,6 +40,7 @@ from ..views import ReportCampaignView from .actions import ( anonymize_action, + delete_asset_with_s3_delete, publish_action, publish_item_action, reopen_asset_action, @@ -411,6 +412,7 @@ class AssetAdmin(admin.ModelAdmin, CustomListDisplayFieldsMixin): unpublish_action, export_to_csv_action, export_to_excel_action, + delete_asset_with_s3_delete, ) autocomplete_fields = ("item",) ordering = ("item__item_id", "sequence") diff --git a/concordia/admin/actions.py b/concordia/admin/actions.py index a3f3c3511..7041d0c91 100644 --- a/concordia/admin/actions.py +++ b/concordia/admin/actions.py @@ -27,16 +27,25 @@ def anonymize_action(modeladmin, request, queryset): def delete_asset_with_s3_delete(modeladmin, request, queryset): + + # Count the number of assets that will become reopened + count = queryset.count() + s3_client = boto3.client("s3") for asset in queryset: response = s3_client.delete_object( - Bucket=settings.S3_BUCKET_NAME, Key=asset.get_storage_url() + Bucket=settings.S3_BUCKET_NAME, + Key=asset.get_storage_url(), ) logger.info(response) asset.delete() + messages.info(request, f"Deleted {count} assets and S3 objects") + -delete_asset_with_s3_delete.short_description = "Delete asset and image files" +delete_asset_with_s3_delete.short_description = ( + "Delete asset, related FK records, and image files" +) def publish_item_action(modeladmin, request, queryset): diff --git a/concordia/models.py b/concordia/models.py index c17e6f2b0..e74eda9d3 100644 --- a/concordia/models.py +++ b/concordia/models.py @@ -109,10 +109,6 @@ def __str__(self): def get_absolute_url(self): return reverse("transcriptions:campaign-detail", args=(self.slug,)) - def get_storage_url(self): - s3_target = "/" + self.slug - return s3_target - class Topic(models.Model): objects = UnlistedPublicationQuerySet.as_manager() @@ -191,10 +187,6 @@ def get_absolute_url(self): kwargs={"campaign_slug": self.campaign.slug, "slug": self.slug}, ) - def get_storage_url(self): - s3_target = "/" + self.campaign.slug + "/" + self.slug - return s3_target - class Item(MetricsModelMixin("item"), models.Model): objects = PublicationQuerySet.as_manager() @@ -234,17 +226,6 @@ def get_absolute_url(self): }, ) - def get_storage_url(self): - s3_target = ( - "/" - + self.project.campaign.slug - + "/" - + self.project.slug - + "/" - + self.item_id - ) - return s3_target - class AssetQuerySet(PublicationQuerySet): def add_contribution_counts(self): @@ -321,25 +302,13 @@ def latest_transcription(self): return self.transcription_set.order_by("-pk").first() def get_storage_url(self): - s3_target = ( - "/" - + self.item.project.campaign.slug - + "/" - + self.item.project.slug - + "/" - + self.item.item_id - + "/" - + self.media_url - ) - return s3_target - - def get_storage_urlv2(self): - return "/".join( + s3_target = [ self.item.project.campaign.slug, self.item.project.slug, self.item.item_id, self.media_url, - ) + ] + return "/".join(s3_target) class Tag(MetricsModelMixin("tag"), models.Model): From df4c83f49eaaac4a683074fe8ffca487a48a8214 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Fri, 7 Jan 2022 10:08:42 -0500 Subject: [PATCH 5/7] inital Asset ImageField post isort black reformat --- concordia/admin/actions.py | 2 +- concordia/models.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/concordia/admin/actions.py b/concordia/admin/actions.py index 7041d0c91..a68648f7d 100644 --- a/concordia/admin/actions.py +++ b/concordia/admin/actions.py @@ -35,7 +35,7 @@ def delete_asset_with_s3_delete(modeladmin, request, queryset): for asset in queryset: response = s3_client.delete_object( Bucket=settings.S3_BUCKET_NAME, - Key=asset.get_storage_url(), + Key=asset.get_storage_key(), ) logger.info(response) asset.delete() diff --git a/concordia/models.py b/concordia/models.py index e74eda9d3..e92c40d7c 100644 --- a/concordia/models.py +++ b/concordia/models.py @@ -1,3 +1,4 @@ +import os.path from logging import getLogger from django.conf import settings @@ -301,7 +302,7 @@ def get_absolute_url(self): def latest_transcription(self): return self.transcription_set.order_by("-pk").first() - def get_storage_url(self): + def get_storage_key(self): s3_target = [ self.item.project.campaign.slug, self.item.project.slug, @@ -310,6 +311,21 @@ def get_storage_url(self): ] return "/".join(s3_target) + def get_storage_path(self, filename): + s3_relative_path = "/".join( + [ + self.item.project.campaign.slug, + self.item.project.slug, + self.item.item_id, + ] + ) + filename = self.media_url + return os.path.join(s3_relative_path, filename) + + storage_image = models.ImageField( + upload_to=get_storage_path, max_length=255, blank=True, null=True + ) + class Tag(MetricsModelMixin("tag"), models.Model): TAG_VALIDATOR = RegexValidator(r"^[- _À-ž'\w]{1,50}$") From 46fd92ef73af44560b23081243697697db98d38a Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Thu, 20 Jan 2022 14:33:06 -0500 Subject: [PATCH 6/7] post isort and black updates --- .../migrations/0051_asset_storage_image.py | 25 +++++++++++++++++++ concordia/signals/handlers.py | 7 +++++- importer/tasks.py | 8 ++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 concordia/migrations/0051_asset_storage_image.py diff --git a/concordia/migrations/0051_asset_storage_image.py b/concordia/migrations/0051_asset_storage_image.py new file mode 100644 index 000000000..97067b34c --- /dev/null +++ b/concordia/migrations/0051_asset_storage_image.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.24 on 2022-01-11 18:14 + +from django.db import migrations, models + +import concordia.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("concordia", "0050_auto_20210920_1544"), + ] + + operations = [ + migrations.AddField( + model_name="asset", + name="storage_image", + field=models.ImageField( + blank=True, + max_length=255, + null=True, + upload_to=concordia.models.Asset.get_storage_path, + ), + ), + ] diff --git a/concordia/signals/handlers.py b/concordia/signals/handlers.py index abf8dd022..772331152 100644 --- a/concordia/signals/handlers.py +++ b/concordia/signals/handlers.py @@ -8,7 +8,7 @@ from django.contrib.auth.models import Group from django.contrib.auth.signals import user_logged_in, user_login_failed from django.core.mail import EmailMultiAlternatives -from django.db.models.signals import post_save +from django.db.models.signals import post_delete, post_save from django.dispatch import receiver from django.template import loader from django_registration.signals import user_activated, user_registered @@ -159,3 +159,8 @@ def send_asset_reservation_message( "sent": time(), }, ) + + +@receiver(post_delete, sender=Asset) +def remove_file_from_s3(sender, instance, using, **kwargs): + instance.storage_image.delete(save=False) diff --git a/importer/tasks.py b/importer/tasks.py index 9f6b242c7..925215cb1 100644 --- a/importer/tasks.py +++ b/importer/tasks.py @@ -401,6 +401,13 @@ def import_item(self, import_item): asset_urls, item_resource_url = get_asset_urls_from_item_resources( import_item.item.metadata.get("resources", []) ) + relative_asset_file_path = "/".join( + [ + import_item.item.project.campaign.slug, + import_item.item.project.slug, + import_item.item.item_id, + ] + ) for idx, asset_url in enumerate(asset_urls, start=1): asset_title = f"{import_item.item.item_id}-{idx}" @@ -413,6 +420,7 @@ def import_item(self, import_item): media_type=MediaType.IMAGE, download_url=asset_url, resource_url=item_resource_url, + storage_image="/".join([relative_asset_file_path, f"{idx}.jpg"]), ) item_asset.full_clean() item_assets.append(item_asset) From 59c010110dc798d3ce8ab1d819358d8529342b16 Mon Sep 17 00:00:00 2001 From: Jennifer Kuenning Date: Fri, 21 Jan 2022 14:46:09 -0500 Subject: [PATCH 7/7] post isort modification --- concordia/admin/__init__.py | 2 -- concordia/admin/actions.py | 24 ------------------------ concordia/models.py | 9 --------- 3 files changed, 35 deletions(-) diff --git a/concordia/admin/__init__.py b/concordia/admin/__init__.py index 282716e68..089b7d199 100644 --- a/concordia/admin/__init__.py +++ b/concordia/admin/__init__.py @@ -40,7 +40,6 @@ from ..views import ReportCampaignView from .actions import ( anonymize_action, - delete_asset_with_s3_delete, publish_action, publish_item_action, reopen_asset_action, @@ -412,7 +411,6 @@ class AssetAdmin(admin.ModelAdmin, CustomListDisplayFieldsMixin): unpublish_action, export_to_csv_action, export_to_excel_action, - delete_asset_with_s3_delete, ) autocomplete_fields = ("item",) ordering = ("item__item_id", "sequence") diff --git a/concordia/admin/actions.py b/concordia/admin/actions.py index a68648f7d..05dd07413 100644 --- a/concordia/admin/actions.py +++ b/concordia/admin/actions.py @@ -1,8 +1,6 @@ import uuid from logging import getLogger -import boto3 -from django.conf import settings from django.contrib import messages from django.utils.timezone import now @@ -26,28 +24,6 @@ def anonymize_action(modeladmin, request, queryset): anonymize_action.short_description = "Anonymize and disable user accounts" -def delete_asset_with_s3_delete(modeladmin, request, queryset): - - # Count the number of assets that will become reopened - count = queryset.count() - - s3_client = boto3.client("s3") - for asset in queryset: - response = s3_client.delete_object( - Bucket=settings.S3_BUCKET_NAME, - Key=asset.get_storage_key(), - ) - logger.info(response) - asset.delete() - - messages.info(request, f"Deleted {count} assets and S3 objects") - - -delete_asset_with_s3_delete.short_description = ( - "Delete asset, related FK records, and image files" -) - - def publish_item_action(modeladmin, request, queryset): """ Mark all of the selected items and their related assets as published diff --git a/concordia/models.py b/concordia/models.py index e92c40d7c..2fdf2db06 100644 --- a/concordia/models.py +++ b/concordia/models.py @@ -302,15 +302,6 @@ def get_absolute_url(self): def latest_transcription(self): return self.transcription_set.order_by("-pk").first() - def get_storage_key(self): - s3_target = [ - self.item.project.campaign.slug, - self.item.project.slug, - self.item.item_id, - self.media_url, - ] - return "/".join(s3_target) - def get_storage_path(self, filename): s3_relative_path = "/".join( [