Merge pull request #139 from DDMAL/img_compression

feat: init img convert & compress for gallery view

pyproject.toml

@@ -12,6 +12,7 @@ django = "4.2.5"
 psycopg = "3.1.10"
 requests = "2.31.0"
 gunicorn = "21.2.0"
+pillow = "^10.4.0"
 
 
 [tool.poetry.group.dev.dependencies]

web-app/django/VIM/apps/instruments/management/commands/download_imgs.py

@@ -0,0 +1,86 @@
+"""This module downloads images from the web and creates thumbnails for the VIM instruments."""
+
+import csv
+import os
+from io import BytesIO
+import requests
+from PIL import Image
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    """Django management command to download images and create thumbnails for instruments."""
+
+    USER_AGENT = "UMIL/0.1.0 (https://vim.simssa.ca/; https://ddmal.music.mcgill.ca/)"
+    OUTPUT_DIR = os.path.join(
+        settings.STATIC_ROOT, "instruments", "images", "instrument_imgs"
+    )
+    CSV_PATH = "startup_data/vim_instruments_with_images-15sept.csv"
+
+    help = "Download images and create thumbnails for instruments"
+
+    def __init__(self):
+        super().__init__()
+        self.headers = {"User-Agent": self.USER_AGENT}
+        self.original_img_dir = os.path.join(self.OUTPUT_DIR, "original")
+        self.thumbnail_dir = os.path.join(self.OUTPUT_DIR, "thumbnail")
+        os.makedirs(self.original_img_dir, exist_ok=True)
+        os.makedirs(self.thumbnail_dir, exist_ok=True)
+
+    def download_image_as_png(self, url, save_path):
+        """Download an image from a URL and save it as a PNG file."""
+        try:
+            response = requests.get(url, stream=True, headers=self.headers, timeout=10)
+            response.raise_for_status()  # Raise an HTTPError for bad responses
+            self._save_image_as_png(response.content, url, save_path)
+        except requests.RequestException as e:
+            self.stderr.write(f"Failed to download image from {url}: {e}")
+
+    def _save_image_as_png(self, img_content, url, save_path):
+        """Save image content as a PNG file."""
+        try:
+            img = Image.open(BytesIO(img_content))
+            img.save(save_path, "PNG")
+            self.stdout.write(f"Saved image at {save_path}")
+        except IOError as e:
+            self.stderr.write(f"Failed to save image from {url}: {e}")
+
+    def create_thumbnail(self, image_path, thumbnail_path, compression_ratio=0.35):
+        """Create a thumbnail of an image."""
+        try:
+            with Image.open(image_path) as original_img:
+                new_size = (
+                    int(original_img.width * compression_ratio),
+                    int(original_img.height * compression_ratio),
+                )
+                original_img.thumbnail(new_size)
+                original_img.save(thumbnail_path, "PNG")
+            self.stdout.write(f"Created thumbnail at {thumbnail_path}")
+        except IOError as e:
+            self.stderr.write(f"Failed to create thumbnail for {image_path}: {e}")
+
+    def process_images(self, csv_file_path):
+        """Process images from a CSV file."""
+        with open(csv_file_path, encoding="utf-8-sig") as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                image_url = row["image"]
+                instrument_wikidata_id = row["instrument"].split("/")[-1]
+                save_path_png = os.path.join(
+                    self.original_img_dir, f"{instrument_wikidata_id}.png"
+                )
+                thumbnail_path = os.path.join(
+                    self.thumbnail_dir, f"{instrument_wikidata_id}.png"
+                )
+
+                if not os.path.exists(save_path_png):
+                    self.download_image_as_png(image_url, save_path_png)
+
+                if not os.path.exists(thumbnail_path) and os.path.exists(save_path_png):
+                    self.create_thumbnail(save_path_png, thumbnail_path)
+
+    def handle(self, *args, **options):
+        """Handle the command."""
+        self.process_images(self.CSV_PATH)
+        self.stdout.write("Images downloaded and thumbnails created")

web-app/django/VIM/apps/instruments/management/commands/import_instruments.py

@@ -1,4 +1,7 @@
+"""This module imports instrument objects from Wikidata for the VIM project."""
+
 import csv
+import os
 from typing import Optional
 import requests
 from django.core.management.base import BaseCommand
@@ -17,6 +20,10 @@ class Command(BaseCommand):
 
     help = "Imports instrument objects"
 
+    def __init__(self):
+        super().__init__()
+        self.language_map: dict[str, Language] = {}
+
     def parse_instrument_data(
         self, instrument_id: str, instrument_data: dict
     ) -> dict[str, str | dict[str, str]]:
@@ -84,14 +91,17 @@ def get_instrument_data(self, instrument_ids: list[str]) -> list[dict]:
         ]
         return instrument_data
 
-    def create_database_objects(self, instrument_attrs: dict, ins_img_url: str) -> None:
+    def create_database_objects(
+        self, instrument_attrs: dict, original_img_path: str, thumbnail_img_path: str
+    ) -> None:
         """
         Given a dictionary of instrument attributes and a url to an instrument image,
         create the corresponding database objects.
 
         instrument_attrs [dict]: Dictionary of instrument attributes. See
             parse_instrument_data for details.
-        ins_img_url [str]: URL of instrument image
+        original_img_path [str]: Path to the original instrument image
+        thumbnail_img_path [str]: Path to the thumbnail of the instrument image
         """
         ins_names = instrument_attrs.pop("ins_names")
         instrument = Instrument.objects.create(**instrument_attrs)
@@ -105,10 +115,17 @@ def create_database_objects(self, instrument_attrs: dict, ins_img_url: str) -> N
         img_obj = AVResource.objects.create(
             instrument=instrument,
             type="image",
-            format=ins_img_url.split(".")[-1],
-            url=ins_img_url,
+            format=original_img_path.split(".")[-1],
+            url=original_img_path,
         )
         instrument.default_image = img_obj
+        thumbnail_obj = AVResource.objects.create(
+            instrument=instrument,
+            type="image",
+            format=thumbnail_img_path.split(".")[-1],
+            url=thumbnail_img_path,
+        )
+        instrument.thumbnail = thumbnail_obj
         instrument.save()
 
     def handle(self, *args, **options) -> None:
@@ -118,15 +135,21 @@ def handle(self, *args, **options) -> None:
             reader = csv.DictReader(csvfile)
             instrument_list: list[dict] = list(reader)
         self.language_map = Language.objects.in_bulk(field_name="wikidata_code")
+        img_dir = "instruments/images/instrument_imgs"
         with transaction.atomic():
             for ins_i in range(0, len(instrument_list), 50):
                 ins_ids_subset: list[str] = [
                     ins["instrument"].split("/")[-1]
                     for ins in instrument_list[ins_i : ins_i + 50]
                 ]
                 ins_data: list[dict] = self.get_instrument_data(ins_ids_subset)
-                ins_imgs_subset: list[str] = [
-                    ins["image"] for ins in instrument_list[ins_i : ins_i + 50]
-                ]
-                for instrument_attrs, ins_img_url in zip(ins_data, ins_imgs_subset):
-                    self.create_database_objects(instrument_attrs, ins_img_url)
+                for instrument_attrs, ins_id in zip(ins_data, ins_ids_subset):
+                    original_img_path = os.path.join(
+                        img_dir, "original", f"{ins_id}.png"
+                    )
+                    thumbnail_img_path = os.path.join(
+                        img_dir, "thumbnail", f"{ins_id}.png"
+                    )
+                    self.create_database_objects(
+                        instrument_attrs, original_img_path, thumbnail_img_path
+                    )

web-app/django/VIM/apps/instruments/management/commands/import_languages.py

@@ -1,3 +1,5 @@
+"""This module imports possible languages for instrument names from Wikidata."""
+
 from django.core.management.base import BaseCommand
 from VIM.apps.instruments.models import Language
 

web-app/django/VIM/apps/instruments/management/commands/index_data.py

@@ -1,8 +1,10 @@
+"""This module indexes instrument data in the database in Solr."""
+
 from django.core.management.base import BaseCommand
 from django.db.models import F, CharField, Value as V
 from django.db.models.functions import Concat, Left
-from VIM.apps.instruments.models import Instrument
 import requests
+from VIM.apps.instruments.models import Instrument
 
 
 class Command(BaseCommand):
@@ -43,6 +45,7 @@ def handle(self, *args, **options):
         )
 
     def build_hbs_label_map(self):
+        """Build a mapping of Hornbostel-Sachs classification codes to labels."""
         # For now, we just want the names in English and French of the first category of
         # the Hornbostel-Sachs classification.
         eng_name_mapping = {

web-app/django/VIM/apps/instruments/migrations/0002_instrument_thumbnail.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.5 on 2024-08-13 14:23
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("instruments", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="instrument",
+            name="thumbnail",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="thumbnail_of",
+                to="instruments.avresource",
+            ),
+        ),
+    ]

web-app/django/VIM/apps/instruments/models/instrument.py

@@ -10,6 +10,13 @@ class Instrument(models.Model):
         null=True,
         related_name="default_image_of",
     )
+    thumbnail = models.ForeignKey(
+        "AVResource",
+        on_delete=models.PROTECT,
+        blank=True,
+        null=True,
+        related_name="thumbnail_of",
+    )
     hornbostel_sachs_class = models.CharField(
         max_length=50, blank=True, help_text="Hornbostel-Sachs classification"
     )

web-app/django/VIM/apps/instruments/templates/instruments/includes/listView.html

@@ -6,7 +6,7 @@
       <div class="row g-0 p-2">
         <div class="col-md-2 align-items-center list-img-container">
           <a href="#" class="text-decoration-none">
-            <img src="{{ instrument.avresource_set.first.url }}" class="img-fluid rounded" alt="instrument image" onerror="this.onerror=null;this.src='{% static "instruments/images/no-image.svg" %}';" />
+            <img src="{% static instrument.thumbnail.url %}" class="img-fluid rounded" alt="instrument thumbnail" onerror="this.onerror=null;this.src='{% static "instruments/images/no-image.svg" %}';" />
           </a>
         </div>
         <div class="col-md-10 card-body pb-2 pt-0">

web-app/django/VIM/apps/instruments/templates/instruments/includes/masonryView.html

@@ -8,7 +8,7 @@
          class="text-decoration-none"
          target="_blank">
         <div class="card mb-3">
-          <img src="{{ instrument.avresource_set.first.url }}" class="card-img-top rounded p-2" alt="instrument image" onerror="this.onerror=null;this.src='{% static "instruments/images/no-image.svg" %}';" />
+          <img src="{% static instrument.thumbnail.url %}" class="card-img-top rounded p-2" alt="instrument thumbnail" onerror="this.onerror=null;this.src='{% static "instruments/images/no-image.svg" %}';" />
           <div class="card-body pb-0 pt-0">
             <p class="card-title text-center notranslate ">
               {% for instrumentname in instrument.instrumentname_set.all %}

web-app/django/VIM/apps/instruments/templates/instruments/includes/stdView.html

@@ -10,7 +10,7 @@
          target="_blank">
         <div class="card mb-3">
           <div class="square-box">
-            <img src="{{ instrument.avresource_set.first.url }}" class="card-img-top rounded p-2 img-fluid" alt="instrument image" onerror="this.onerror=null;this.src='{% static "instruments/images/no-image.svg" %}';" />
+            <img src="{% static instrument.thumbnail.url %}" class="card-img-top rounded p-2 img-fluid" alt="instrument thumbnail" onerror="this.onerror=null;this.src='{% static "instruments/images/no-image.svg" %}';" />
           </div>
           <div class="card-body pb-0 pt-0">
             <p class="card-title text-center notranslate">