From c14de8a2756a6c9e566b8ff230d79cb95c8c2469 Mon Sep 17 00:00:00 2001 From: Kun Fang Date: Tue, 13 Aug 2024 14:47:34 -0400 Subject: [PATCH] fix: management commands & migrations fix - Add `__init__.py` under `VIM/apps` directory to fix pylint error: `No name 'apps' in module 'VIM' Pylint(E0611:no-name-in-module)`; - Adjust importing order in all management commands; - In `download_imgs.py`: (1) remove class `ImageDownloader` and place all methods into `Command` class; (2) change constants into upper case; (3) add `timeout` for `requests.get`; (4) change general `Exception` into specific exception types such as `requests.RequestException` and `IOError`; (5) change all `print()` into `self.stdout.write` and `self.stderr.write`; - In `import_instruments.py`, change relative image path into absolute path; - Recover migrations file; create an additional file for thumbnail field; Refs: #138 --- web-app/django/VIM/apps/__init__.py | 0 .../management/commands/download_imgs.py | 70 ++++++++++--------- .../management/commands/import_instruments.py | 20 +++++- .../management/commands/import_languages.py | 2 + .../management/commands/index_data.py | 5 +- .../instruments/migrations/0001_initial.py | 12 +--- .../migrations/0002_instrument_thumbnail.py | 24 +++++++ 7 files changed, 86 insertions(+), 47 deletions(-) create mode 100644 web-app/django/VIM/apps/__init__.py create mode 100644 web-app/django/VIM/apps/instruments/migrations/0002_instrument_thumbnail.py diff --git a/web-app/django/VIM/apps/__init__.py b/web-app/django/VIM/apps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/web-app/django/VIM/apps/instruments/management/commands/download_imgs.py b/web-app/django/VIM/apps/instruments/management/commands/download_imgs.py index 53b26ac..b35d2dd 100644 --- a/web-app/django/VIM/apps/instruments/management/commands/download_imgs.py +++ b/web-app/django/VIM/apps/instruments/management/commands/download_imgs.py @@ -1,35 +1,50 @@ +"""This module downloads images from the web and creates thumbnails for the VIM instruments.""" + import csv import os +from io import BytesIO import requests from PIL import Image -from io import BytesIO from django.core.management.base import BaseCommand -class ImageDownloader: - def __init__(self, user_agent, output_dir): - self.headers = {"User-Agent": user_agent} - self.original_img_dir = os.path.join(output_dir, "original") - self.thumbnail_dir = os.path.join(output_dir, "thumbnail") +class Command(BaseCommand): + """Django management command to download images and create thumbnails for instruments.""" + + USER_AGENT = "UMIL/0.1.0 (https://vim.simssa.ca/; https://ddmal.music.mcgill.ca/)" + OUTPUT_DIR = "VIM/apps/instruments/static/instruments/images/instrument_imgs" + CSV_PATH = "startup_data/vim_instruments_with_images-15sept.csv" + + help = "Download images and create thumbnails for instruments" + + def __init__(self): + super().__init__() + self.headers = {"User-Agent": self.USER_AGENT} + self.original_img_dir = os.path.join(self.OUTPUT_DIR, "original") + self.thumbnail_dir = os.path.join(self.OUTPUT_DIR, "thumbnail") os.makedirs(self.original_img_dir, exist_ok=True) os.makedirs(self.thumbnail_dir, exist_ok=True) def download_image_as_png(self, url, save_path): + """Download an image from a URL and save it as a PNG file.""" try: - response = requests.get(url, stream=True, headers=self.headers) + response = requests.get(url, stream=True, headers=self.headers, timeout=10) response.raise_for_status() # Raise an HTTPError for bad responses - self._save_image_as_png(response.content, save_path) - print(f"Downloaded {url} to {save_path}") + self._save_image_as_png(response.content, url, save_path) except requests.RequestException as e: - print(f"Failed to download {url}: {e}") - except Exception as e: - print(f"Error processing {url}: {e}") + self.stderr.write(f"Failed to download image from {url}: {e}") - def _save_image_as_png(self, img_content, save_path): - img = Image.open(BytesIO(img_content)) - img.save(save_path, "PNG") + def _save_image_as_png(self, img_content, url, save_path): + """Save image content as a PNG file.""" + try: + img = Image.open(BytesIO(img_content)) + img.save(save_path, "PNG") + self.stdout.write(f"Saved image at {save_path}") + except IOError as e: + self.stderr.write(f"Failed to save image from {url}: {e}") def create_thumbnail(self, image_path, thumbnail_path, compression_ratio=0.35): + """Create a thumbnail of an image.""" try: with Image.open(image_path) as original_img: new_size = ( @@ -38,11 +53,12 @@ def create_thumbnail(self, image_path, thumbnail_path, compression_ratio=0.35): ) original_img.thumbnail(new_size) original_img.save(thumbnail_path, "PNG") - print(f"Created thumbnail for {image_path}") - except Exception as e: - print(f"Error creating thumbnail for {image_path}: {e}") + self.stdout.write(f"Created thumbnail at {thumbnail_path}") + except IOError as e: + self.stderr.write(f"Failed to create thumbnail for {image_path}: {e}") def process_images(self, csv_file_path): + """Process images from a CSV file.""" with open(csv_file_path, encoding="utf-8-sig") as csvfile: reader = csv.DictReader(csvfile) for row in reader: @@ -58,20 +74,10 @@ def process_images(self, csv_file_path): if not os.path.exists(save_path_png): self.download_image_as_png(image_url, save_path_png) - if not os.path.exists(thumbnail_path): + if not os.path.exists(thumbnail_path) and os.path.exists(save_path_png): self.create_thumbnail(save_path_png, thumbnail_path) - -class Command(BaseCommand): - help = "Download images and create thumbnails for instruments" - def handle(self, *args, **options): - user_agent = ( - "UMIL/0.1.0 (https://vim.simssa.ca/; https://ddmal.music.mcgill.ca/)" - ) - output_dir = "VIM/apps/instruments/static/instruments/images/instrument_imgs" - csv_file_path = "startup_data/vim_instruments_with_images-15sept.csv" - - downloader = ImageDownloader(user_agent, output_dir) - downloader.process_images(csv_file_path) - print("Images downloaded and thumbnails created") + """Handle the command.""" + self.process_images(self.CSV_PATH) + self.stdout.write("Images downloaded and thumbnails created") diff --git a/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py b/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py index 8f663fd..b82b8ff 100644 --- a/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py +++ b/web-app/django/VIM/apps/instruments/management/commands/import_instruments.py @@ -1,8 +1,12 @@ +"""This module imports instrument objects from Wikidata for the VIM project.""" + import csv +import os from typing import Optional import requests from django.core.management.base import BaseCommand from django.db import transaction +from django.conf import settings from VIM.apps.instruments.models import Instrument, InstrumentName, Language, AVResource @@ -17,6 +21,10 @@ class Command(BaseCommand): help = "Imports instrument objects" + def __init__(self): + super().__init__() + self.language_map: dict[str, Language] = {} + def parse_instrument_data( self, instrument_id: str, instrument_data: dict ) -> dict[str, str | dict[str, str]]: @@ -128,6 +136,9 @@ def handle(self, *args, **options) -> None: reader = csv.DictReader(csvfile) instrument_list: list[dict] = list(reader) self.language_map = Language.objects.in_bulk(field_name="wikidata_code") + img_dir = os.path.join( + settings.STATIC_URL, "instruments", "images", "instrument_imgs" + ) with transaction.atomic(): for ins_i in range(0, len(instrument_list), 50): ins_ids_subset: list[str] = [ @@ -136,9 +147,12 @@ def handle(self, *args, **options) -> None: ] ins_data: list[dict] = self.get_instrument_data(ins_ids_subset) for instrument_attrs, ins_id in zip(ins_data, ins_ids_subset): - img_dir = "../../static/instruments/images/instrument_imgs" - original_img_path = f"{img_dir}/original/{ins_id}.png" - thumbnail_img_path = f"{img_dir}/thumbnail/{ins_id}.png" + original_img_path = os.path.join( + img_dir, "original", f"{ins_id}.png" + ) + thumbnail_img_path = os.path.join( + img_dir, "thumbnail", f"{ins_id}.png" + ) self.create_database_objects( instrument_attrs, original_img_path, thumbnail_img_path ) diff --git a/web-app/django/VIM/apps/instruments/management/commands/import_languages.py b/web-app/django/VIM/apps/instruments/management/commands/import_languages.py index da9d52d..17c2fbf 100644 --- a/web-app/django/VIM/apps/instruments/management/commands/import_languages.py +++ b/web-app/django/VIM/apps/instruments/management/commands/import_languages.py @@ -1,3 +1,5 @@ +"""This module imports possible languages for instrument names from Wikidata.""" + from django.core.management.base import BaseCommand from VIM.apps.instruments.models import Language diff --git a/web-app/django/VIM/apps/instruments/management/commands/index_data.py b/web-app/django/VIM/apps/instruments/management/commands/index_data.py index 56cd10f..d613ea4 100644 --- a/web-app/django/VIM/apps/instruments/management/commands/index_data.py +++ b/web-app/django/VIM/apps/instruments/management/commands/index_data.py @@ -1,8 +1,10 @@ +"""This module indexes instrument data in the database in Solr.""" + from django.core.management.base import BaseCommand from django.db.models import F, CharField, Value as V from django.db.models.functions import Concat, Left -from VIM.apps.instruments.models import Instrument import requests +from VIM.apps.instruments.models import Instrument class Command(BaseCommand): @@ -43,6 +45,7 @@ def handle(self, *args, **options): ) def build_hbs_label_map(self): + """Build a mapping of Hornbostel-Sachs classification codes to labels.""" # For now, we just want the names in English and French of the first category of # the Hornbostel-Sachs classification. eng_name_mapping = { diff --git a/web-app/django/VIM/apps/instruments/migrations/0001_initial.py b/web-app/django/VIM/apps/instruments/migrations/0001_initial.py index e91acb0..54f5491 100644 --- a/web-app/django/VIM/apps/instruments/migrations/0001_initial.py +++ b/web-app/django/VIM/apps/instruments/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.5 on 2024-08-12 14:23 +# Generated by Django 4.2.5 on 2023-09-27 18:47 from django.db import migrations, models import django.db.models.deletion @@ -102,16 +102,6 @@ class Migration(migrations.Migration): to="instruments.avresource", ), ), - ( - "thumbnail", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.PROTECT, - related_name="thumbnail_of", - to="instruments.avresource", - ), - ), ], ), migrations.CreateModel( diff --git a/web-app/django/VIM/apps/instruments/migrations/0002_instrument_thumbnail.py b/web-app/django/VIM/apps/instruments/migrations/0002_instrument_thumbnail.py new file mode 100644 index 0000000..4463181 --- /dev/null +++ b/web-app/django/VIM/apps/instruments/migrations/0002_instrument_thumbnail.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.5 on 2024-08-13 14:23 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("instruments", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="instrument", + name="thumbnail", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="thumbnail_of", + to="instruments.avresource", + ), + ), + ]