minor snagging

.gitignore

@@ -160,5 +160,40 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
-
-.terraform
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore transient lock info files created by terraform apply
+.terraform.tfstate.lock.info
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

src/transferrer/make_zip.py

@@ -56,11 +56,11 @@ def create_born_digital_folder(source_files, target_directory, accession_id):
 
 def move_files_to_objects_folder(source_files, target_directory):
     """
-    Move the contents of source directory into the objects subfolder in the target directory
+    Move the contents of source directory into the target directory
     """
-    os.makedirs(os.path.join(target_directory, "objects"))
+    os.makedirs(target_directory, exist_ok=True)
     for file_abspath in source_files:
-        shutil.move(file_abspath, os.path.join(target_directory, "objects"))
+        shutil.move(file_abspath, target_directory)
 
 
 def files_in_folder(source_directory):

test/conftest.py

@@ -17,6 +17,18 @@ def moto_s3():
     finally:
         moto_fake.stop()
 
+
+@pytest.fixture
+def moto_session():
+    moto_fake = moto.mock_aws()
+    try:
+        moto_fake.start()
+        yield boto3
+    finally:
+        moto_fake.stop()
+
+
+
 @pytest.fixture
 def empty_bucket(moto_s3):
     @contextmanager

test/test_make_zips.py

@@ -50,13 +50,13 @@ def test_single_zip(fs):
     populate_source_dir_with_images(fs, "G00DCAFE", 2)
     next(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 10))
     # it creates a zip named using the accession and shoot numbers
-    with zipfile.ZipFile("./1234_G00DCAFE.zip") as zip_file:
+    with zipfile.ZipFile("./out/1234_G00DCAFE.zip") as zip_file:
         zip_file.extractall("/unzipped")
         # with a metadata csv in the root of the zip
         assert_csv_has_accession_id("/unzipped/metadata/metadata.csv", "1234_G00DCAFE")
-        # and the photos in an ./objects folder
-        assert os.path.exists("/unzipped/objects/G00DCAFE_001.tif")
-        assert os.path.exists("/unzipped/objects/G00DCAFE_002.tif")
+        # and the photos in an . folder
+        assert os.path.exists("/unzipped/G00DCAFE_001.tif")
+        assert os.path.exists("/unzipped/G00DCAFE_002.tif")
 
 
 def test_multiple_zips(fs):
@@ -66,19 +66,19 @@ def test_multiple_zips(fs):
 
     list(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 10))
     # it creates zips named using the accession and shoot numbers, with a three-digit numeric suffix
-    with zipfile.ZipFile("./1234_G00DCAFE_001.zip") as zip_file:
+    with zipfile.ZipFile("./out/1234_G00DCAFE_001.zip") as zip_file:
         zip_file.extractall("/unzipped_001")
         assert_csv_has_accession_id("/unzipped_001/metadata/metadata.csv", "1234_G00DCAFE_001")
         # The objects chosen for each zip are predictable and consistent.
         # They are sorted alphanumerically before being sliced into groups to place into each zip
-        objects = os.listdir("/unzipped_001/objects")
+        objects = fnmatch.filter(os.listdir("/unzipped_001"), '*tif')
         assert len(objects) == 10
         assert set(filename[:3] for filename in objects) == {"AAA"}
 
-    with zipfile.ZipFile("./1234_G00DCAFE_002.zip") as zip_file:
+    with zipfile.ZipFile("./out/1234_G00DCAFE_002.zip") as zip_file:
         zip_file.extractall("/unzipped_002")
         assert_csv_has_accession_id("/unzipped_002/metadata/metadata.csv", "1234_G00DCAFE_002")
-        objects = os.listdir("/unzipped_002/objects")
+        objects = fnmatch.filter(os.listdir("/unzipped_002"), '*tif')
         assert len(objects) == 10
         assert set(filename[:3] for filename in objects) == {"BBB"}
 
@@ -99,13 +99,13 @@ def test_ignored_files_single_zip(fs):
     populate_source_dir_with_images(fs, "HELLO", 10)
     populate_source_dir(fs, ("shoot.csv", "HELLO.xml"))
     list(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 10))
-    assert fnmatch.filter(os.listdir("."), "*zip") == ["1234_G00DCAFE.zip"]
+    assert fnmatch.filter(os.listdir("./out/"), "*zip") == ["1234_G00DCAFE.zip"]
 
 
 def test_ignored_files_multiple_zips(fs):
     populate_source_dir_with_images(fs, "HELLO", 20)
     populate_source_dir(fs, ("shoot.csv", "HELLO.xml"))
     list(create_born_digital_zips("/in", "/out", "1234", "G00DCAFE", 5))
-    zips = fnmatch.filter(os.listdir("."), "*zip")
+    zips = fnmatch.filter(os.listdir("./out/"), "*zip")
     # 20/5 == 4, ceil 22/5 > 4
     assert len(zips) == 4

test/test_restore.py

@@ -2,7 +2,7 @@
 import pytest
 import pyfakefs
 from transferrer.download import download_shoot_folder
-from transferrer.restore import restore_shoot_folder
+from restore import restore_shoot_folder
 
 
 def test_ignores_metadata_files(glacier_shoot_bucket, fs):

test/test_upload.py

@@ -3,16 +3,17 @@
 import boto3
 from transferrer.upload import upload
 from transferrer.upload import upload
+from moto import mock_aws
 
-
-def test_raises_on_missing_zip(moto_s3, target_bucket, fs):
+@mock_aws
+def test_raises_on_missing_zip(moto_session, target_bucket, fs):
     with pytest.raises(FileNotFoundError):
-        upload(moto_s3, "missing.zip")
-
+        upload(moto_session, "missing.zip")
 
-def test_uploads_to_accessions_folder_in_bucket(moto_s3, target_bucket, fs):
+@mock_aws
+def test_uploads_to_accessions_folder_in_bucket(moto_session, target_bucket, fs):
     fs.create_file("present.zip")
-    upload(moto_s3, "present.zip")
+    upload(moto_session, "present.zip")
     assert [obj.key for obj in target_bucket.objects.all()] == ["born-digital-accessions/present.zip"]