From 350a38655ba9c4e478a1d8f0b313bbcd12f89673 Mon Sep 17 00:00:00 2001 From: Paul Butcher Date: Wed, 17 Jul 2024 15:06:41 +0100 Subject: [PATCH] minor snagging --- .gitignore | 39 ++++++++++++++++++++++++-- src/{transferrer => }/check_status.py | 0 src/{transferrer => }/restore.py | 0 src/transferrer/make_zip.py | 6 ++-- terraform/lambda.zip | Bin 29824 -> 30205 bytes test/conftest.py | 12 ++++++++ test/test_make_zips.py | 20 ++++++------- test/test_restore.py | 2 +- test/test_upload.py | 13 +++++---- 9 files changed, 70 insertions(+), 22 deletions(-) rename src/{transferrer => }/check_status.py (100%) rename src/{transferrer => }/restore.py (100%) diff --git a/.gitignore b/.gitignore index aa3b8f5..4b77d0b 100644 --- a/.gitignore +++ b/.gitignore @@ -160,5 +160,40 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ - -.terraform \ No newline at end of file +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore transient lock info files created by terraform apply +.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/src/transferrer/check_status.py b/src/check_status.py similarity index 100% rename from src/transferrer/check_status.py rename to src/check_status.py diff --git a/src/transferrer/restore.py b/src/restore.py similarity index 100% rename from src/transferrer/restore.py rename to src/restore.py diff --git a/src/transferrer/make_zip.py b/src/transferrer/make_zip.py index 248bad2..bf5e929 100644 --- a/src/transferrer/make_zip.py +++ b/src/transferrer/make_zip.py @@ -56,11 +56,11 @@ def create_born_digital_folder(source_files, target_directory, accession_id): def move_files_to_objects_folder(source_files, target_directory): """ - Move the contents of source directory into the objects subfolder in the target directory + Move the contents of source directory into the target directory """ - os.makedirs(os.path.join(target_directory, "objects")) + os.makedirs(target_directory, exist_ok=True) for file_abspath in source_files: - shutil.move(file_abspath, os.path.join(target_directory, "objects")) + shutil.move(file_abspath, target_directory) def files_in_folder(source_directory): diff --git a/terraform/lambda.zip b/terraform/lambda.zip index 96654cf1c7ea3f84ce6592db5d43b4fd2df274b0..ac648d45a219eaaf6bd165a18caa484fc62daa74 100644 GIT binary patch delta 403 zcmZp8$@up*Pdh3?)U0dBtg|MMbGa`o$%QMJ4etkz&1q%8&_XvtAnrw0^JZ zI;bs~lz9J%nAQj7lB@4pW%^|uZH%mp7UswQab8)z`G-aEIq_wd%Y1&cS)J2+RuSs; z)#ifC#JfitQ$v)Mz~p*~LeKJRtSqiO1uf4#+-Jn} zdF#DhA3WF8A9PBnTp8CBWBM)Z{<^dOch%MZzJ6oQo`eS>x&DEQzYR}Rr0^Ney%jK_ zc=z5r@2idV?;1Z?X#XwnZTt;|$0S4FW%<#9YO-Ll@Z?X$QcV2mlle-V_3vh&hHPm;PJUtvFhB#mS=oS= QFsJ}w1UCaie-?-b0HHXmmH+?% delta 52 zcmezSnz7*}He05@;+>pdDmB@*RABPVQd6ci>60&%%JV5RC^PVIGcd$u HG5`SpQaKU7 diff --git a/test/conftest.py b/test/conftest.py index ddff173..bf30b83 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -17,6 +17,18 @@ def moto_s3(): finally: moto_fake.stop() + +@pytest.fixture +def moto_session(): + moto_fake = moto.mock_aws() + try: + moto_fake.start() + yield boto3 + finally: + moto_fake.stop() + + + @pytest.fixture def empty_bucket(moto_s3): @contextmanager diff --git a/test/test_make_zips.py b/test/test_make_zips.py index 1bfd257..20180ca 100644 --- a/test/test_make_zips.py +++ b/test/test_make_zips.py @@ -50,13 +50,13 @@ def test_single_zip(fs): populate_source_dir_with_images(fs, "G00DCAFE", 2) next(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 10)) # it creates a zip named using the accession and shoot numbers - with zipfile.ZipFile("./1234_G00DCAFE.zip") as zip_file: + with zipfile.ZipFile("./out/1234_G00DCAFE.zip") as zip_file: zip_file.extractall("/unzipped") # with a metadata csv in the root of the zip assert_csv_has_accession_id("/unzipped/metadata/metadata.csv", "1234_G00DCAFE") - # and the photos in an ./objects folder - assert os.path.exists("/unzipped/objects/G00DCAFE_001.tif") - assert os.path.exists("/unzipped/objects/G00DCAFE_002.tif") + # and the photos in an . folder + assert os.path.exists("/unzipped/G00DCAFE_001.tif") + assert os.path.exists("/unzipped/G00DCAFE_002.tif") def test_multiple_zips(fs): @@ -66,19 +66,19 @@ def test_multiple_zips(fs): list(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 10)) # it creates zips named using the accession and shoot numbers, with a three-digit numeric suffix - with zipfile.ZipFile("./1234_G00DCAFE_001.zip") as zip_file: + with zipfile.ZipFile("./out/1234_G00DCAFE_001.zip") as zip_file: zip_file.extractall("/unzipped_001") assert_csv_has_accession_id("/unzipped_001/metadata/metadata.csv", "1234_G00DCAFE_001") # The objects chosen for each zip are predictable and consistent. # They are sorted alphanumerically before being sliced into groups to place into each zip - objects = os.listdir("/unzipped_001/objects") + objects = fnmatch.filter(os.listdir("/unzipped_001"), '*tif') assert len(objects) == 10 assert set(filename[:3] for filename in objects) == {"AAA"} - with zipfile.ZipFile("./1234_G00DCAFE_002.zip") as zip_file: + with zipfile.ZipFile("./out/1234_G00DCAFE_002.zip") as zip_file: zip_file.extractall("/unzipped_002") assert_csv_has_accession_id("/unzipped_002/metadata/metadata.csv", "1234_G00DCAFE_002") - objects = os.listdir("/unzipped_002/objects") + objects = fnmatch.filter(os.listdir("/unzipped_002"), '*tif') assert len(objects) == 10 assert set(filename[:3] for filename in objects) == {"BBB"} @@ -99,13 +99,13 @@ def test_ignored_files_single_zip(fs): populate_source_dir_with_images(fs, "HELLO", 10) populate_source_dir(fs, ("shoot.csv", "HELLO.xml")) list(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 10)) - assert fnmatch.filter(os.listdir("."), "*zip") == ["1234_G00DCAFE.zip"] + assert fnmatch.filter(os.listdir("./out/"), "*zip") == ["1234_G00DCAFE.zip"] def test_ignored_files_multiple_zips(fs): populate_source_dir_with_images(fs, "HELLO", 20) populate_source_dir(fs, ("shoot.csv", "HELLO.xml")) list(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 5)) - zips = fnmatch.filter(os.listdir("."), "*zip") + zips = fnmatch.filter(os.listdir("./out/"), "*zip") # 20/5 == 4, ceil 22/5 > 4 assert len(zips) == 4 diff --git a/test/test_restore.py b/test/test_restore.py index 7605b73..d17e995 100644 --- a/test/test_restore.py +++ b/test/test_restore.py @@ -2,7 +2,7 @@ import pytest import pyfakefs from transferrer.download import download_shoot_folder -from transferrer.restore import restore_shoot_folder +from restore import restore_shoot_folder def test_ignores_metadata_files(glacier_shoot_bucket, fs): diff --git a/test/test_upload.py b/test/test_upload.py index 146c9f7..fc31938 100644 --- a/test/test_upload.py +++ b/test/test_upload.py @@ -3,16 +3,17 @@ import boto3 from transferrer.upload import upload from transferrer.upload import upload +from moto import mock_aws - -def test_raises_on_missing_zip(moto_s3, target_bucket, fs): +@mock_aws +def test_raises_on_missing_zip(moto_session, target_bucket, fs): with pytest.raises(FileNotFoundError): - upload(moto_s3, "missing.zip") - + upload(moto_session, "missing.zip") -def test_uploads_to_accessions_folder_in_bucket(moto_s3, target_bucket, fs): +@mock_aws +def test_uploads_to_accessions_folder_in_bucket(moto_session, target_bucket, fs): fs.create_file("present.zip") - upload(moto_s3, "present.zip") + upload(moto_session, "present.zip") assert [obj.key for obj in target_bucket.objects.all()] == ["born-digital-accessions/present.zip"]