deploy with Terraform (#20)

* deploy with Terraform

* Add Makefile (#21)

* Add Makefile

* set the splitter to 20

* neater permission

Makefile

@@ -0,0 +1,49 @@
+.PHONY: shoots/clean %.sliced
+.SECONDARY:
+
+# Remove intermediate/final files from the shoots folder
+shoots/clean:
+	rm shoots/*restored
+	rm shoots/*transferred
+	rm shoots/*slice*
+
+# Request the Glacier restoration of the shoots in the given file
+# The file is expected to contain one shoot identifier per line.
+# In order to run this, set your AWS profile to one with authority in the workflow account.
+%.restored : %
+	cat $< | python src/restore.py
+	cp $< $@
+
+
+# Request the Glacier transfer of the shoots in the given file
+# This rule depends on restoration having completed, which is not guaranteed
+# (or even likely) if you run this rule without having previously requested the restoration
+# Any shoots that are not yet fully restored will result in a DLQ message that can eventually
+# be redriven when the s3 objects are finally available for download
+# In order to run this, set your AWS profile to one with authority in the digitisation account.
+
+# transfer to staging (see above)
+%.transferred.staging: %.restored
+	cat $< | python src/start_transfers.py staging
+	cp $< $@
+
+
+# transfer to production (see above)
+%.transferred.production: %.restored
+	cat $< | python src/start_transfers.py production
+	cp $< $@
+
+# Slice a given input file into manageable chunks, so that you can run them through the
+# transfer process separately without overwhelming the target system.
+# The right number for archivematica is probably about 20.
+
+%.sliced: %
+	split -l 20 $< $<.
+
+# Touch the files already on AWS.  This will stimulate the corresponding transfer lambdas
+# In order to run this, set your AWS profile to one with authority in the digitisation account.
+%.touched.staging: %
+	cat % | python src/touch.py staging
+
+%.touched.production: %
+	cat % | python src/touch.py production

terraform/main.tf

@@ -0,0 +1,36 @@
+locals {
+  event_batching_window_timeout = 20
+  lambda_timeout = 120 //two minutes
+
+  # The lambda event source pulls messages from SQS in batches, finally triggering the lambda
+  # when either it has enough messages, or enough time has elapsed.
+  # A message becomes invisible when it joins the event source buffer, so could wait for
+  # the whole timeout window plus the whole execution time before being confirmed.
+  # The value of visibility timeout must be at least 20 seconds more than the lambda timeout
+  # This doesn't necessarily need to exist with a longer batching window, but
+  # always adding 20 here should mean that you can safely set batching window to 0
+  # if you wish.
+  # See: https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html
+  # "Lambda might wait for up to 20 seconds before invoking your function."
+  queue_visibility_timeout = local.event_batching_window_timeout + local.lambda_timeout + 20
+}
+
+data "archive_file" "lambda_zip" {
+  type        = "zip"
+  output_path = "lambda.zip"
+  source_dir = "../src"
+}
+
+module "staging_lambda" {
+  source = "./modules/transferrer_pipe"
+  environment = "staging"
+  queue_visibility_timeout = local.queue_visibility_timeout
+  lambda_zip = data.archive_file.lambda_zip
+}
+
+module "production_lambda" {
+  source = "./modules/transferrer_pipe"
+  environment = "production"
+  queue_visibility_timeout = local.queue_visibility_timeout
+  lambda_zip = data.archive_file.lambda_zip
+}

terraform/modules/notification_queue/main.tf

@@ -0,0 +1,22 @@
+
+module "transfer_shoots_topic" {
+  source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.1"
+  name = "transfer-shoots-${var.environment}"
+}
+
+module "dlq_alarm_topic" {
+  source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.1"
+  name = "transfer-shoots-alarm-${var.environment}"
+}
+
+module "input_queue" {
+  source = "github.com/wellcomecollection/terraform-aws-sqs//queue?ref=v1.2.1"
+
+  queue_name = "transfer-shoots-${var.environment}"
+
+  topic_arns                 = [module.transfer_shoots_topic.arn]
+  visibility_timeout_seconds = var.queue_visibility_timeout
+  max_receive_count          = 1
+  message_retention_seconds = 1200
+  alarm_topic_arn           = module.dlq_alarm_topic.arn
+}

terraform/modules/notification_queue/outputs.tf

@@ -0,0 +1,3 @@
+output "queue_arn" {
+  value = module.input_queue.arn
+}

terraform/modules/notification_queue/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/notification_queue/variables.tf

@@ -0,0 +1,11 @@
+variable "queue_visibility_timeout" {
+  type = number
+}
+
+variable "environment" {
+  type = string
+  validation {
+    condition = contains(["staging", "production"], var.environment)
+    error_message = "environment must be one of staging or production"
+  }
+}

terraform/modules/sqs_lambda_trigger/main.tf

@@ -0,0 +1,33 @@
+
+data "aws_iam_policy_document" "allow_sqs_pull" {
+    statement {
+      actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes"
+    ]
+    resources = [
+      var.queue_arn
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "allow_sqs_pull" {
+  name   = "${var.trigger_name}-pull-from-queue"
+  role   = var.role_name
+  policy = data.aws_iam_policy_document.allow_sqs_pull.json
+}
+
+resource "aws_lambda_event_source_mapping" "lambda_trigger" {
+  event_source_arn = var.queue_arn
+  enabled          = true
+  function_name    = var.function_name
+  batch_size       = var.batch_size
+}
+
+resource "aws_lambda_permission" "allow_lambda_sqs_trigger" {
+  action        = "lambda:InvokeFunction"
+  function_name    = var.function_name
+  principal     = "sqs.amazonaws.com"
+  source_arn = var.queue_arn
+}

terraform/modules/sqs_lambda_trigger/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/sqs_lambda_trigger/variables.tf

@@ -0,0 +1,19 @@
+variable "queue_arn" {
+  type = string
+}
+
+variable "function_name" {
+  type = string
+}
+
+variable "role_name" {
+    type = string
+}
+
+variable "trigger_name" {
+  type = string
+}
+variable "batch_size" {
+  type = number
+  default = 1
+}

terraform/modules/transferrer_lambda/main.tf

@@ -0,0 +1,74 @@
+locals {
+  lambda_name = "editorial-photography-transfer-${var.environment}"
+  lambda_timeout = 300 //five minutes
+  buckets = tomap(
+    {
+      staging = "wellcomecollection-archivematica-staging-transfer-source",
+      production = "wellcomecollection-archivematica-transfer-source"
+    }
+  )
+  target_bucket = lookup(local.buckets, var.environment)
+
+}
+
+
+module "transfer_lambda" {
+  source = "git@github.com:wellcomecollection/terraform-aws-lambda?ref=v1.2.0"
+
+  name    = local.lambda_name
+  runtime = "python3.12"
+  handler = "lambda_function.lambda_handler"
+
+  filename    = var.lambda_zip.output_path
+  memory_size = 2048
+  timeout     = local.lambda_timeout
+
+  environment = {
+    variables = {
+      ACCESSION_NUMBER = "2754"
+      TARGET_BUCKET = local.target_bucket
+    }
+  }
+  source_code_hash = var.lambda_zip.output_base64sha256
+  ephemeral_storage  = {
+    size = 4096
+  }
+}
+
+resource "aws_iam_role_policy" "write_to_archivematica_transfer_source" {
+  role = module.transfer_lambda.lambda_role.name
+  name = "write_to_archivematica_transfer_source-${var.environment}"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+            "Effect": "Allow",
+            "Action": "s3:PutObject",
+            "Resource": "arn:aws:s3:::${local.target_bucket}/*"
+        },
+    ]
+  }
+  )
+}
+
+resource "aws_iam_role_policy" "read_from_editorial_photography" {
+  role = module.transfer_lambda.lambda_role.name
+  name = "read_from_editorial_photography-${var.environment}"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+            "Effect" =  "Allow",
+            "Action" =  [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource" = [
+              "arn:aws:s3:::wellcomecollection-editorial-photography",
+              "arn:aws:s3:::wellcomecollection-editorial-photography/*"
+            ],
+        },
+    ]
+  })
+
+}

terraform/modules/transferrer_lambda/outputs.tf

@@ -0,0 +1,7 @@
+output "lambda" {
+  value = module.transfer_lambda.lambda
+}
+
+output "role" {
+  value = module.transfer_lambda.lambda_role
+}

terraform/modules/transferrer_lambda/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/transferrer_lambda/variables.tf

@@ -0,0 +1,16 @@
+variable "environment" {
+  type = string
+  validation {
+    condition = contains(["staging", "production"], var.environment)
+    error_message = "environment must be one of staging or production"
+  }
+}
+
+variable "lambda_zip" {
+  type = object(
+    {
+      output_path = string,
+      output_base64sha256 = string
+    }
+  )
+}

terraform/modules/transferrer_pipe/main.tf

@@ -0,0 +1,20 @@
+
+module "transfer_lambda" {
+  source = "../transferrer_lambda"
+  environment = var.environment
+  lambda_zip = var.lambda_zip
+}
+
+module "input_queue" {
+  source = "../notification_queue"
+  environment = var.environment
+  queue_visibility_timeout = var.queue_visibility_timeout
+}
+
+module "trigger" {
+  source = "../sqs_lambda_trigger"
+  queue_arn = module.input_queue.queue_arn
+  function_name = module.transfer_lambda.lambda.function_name
+  role_name = module.transfer_lambda.role.name
+  trigger_name = "editorial-photography-${var.environment}"
+}

terraform/modules/transferrer_pipe/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}