terraform the infrastructure

.gitignore

@@ -159,4 +159,6 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-.idea/
+.idea/
+
+.terraform

terraform/main.tf

@@ -0,0 +1,36 @@
+locals {
+  event_batching_window_timeout = 20
+  lambda_timeout = 120 //two minutes
+
+  # The lambda event source pulls messages from SQS in batches, finally triggering the lambda
+  # when either it has enough messages, or enough time has elapsed.
+  # A message becomes invisible when it joins the event source buffer, so could wait for
+  # the whole timeout window plus the whole execution time before being confirmed.
+  # The value of visibility timeout must be at least 20 seconds more than the lambda timeout
+  # This doesn't necessarily need to exist with a longer batching window, but
+  # always adding 20 here should mean that you can safely set batching window to 0
+  # if you wish.
+  # See: https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html
+  # "Lambda might wait for up to 20 seconds before invoking your function."
+  queue_visibility_timeout = local.event_batching_window_timeout + local.lambda_timeout + 20
+}
+
+data "archive_file" "lambda_zip" {
+  type        = "zip"
+  output_path = "lambda.zip"
+  source_dir = "../src"
+}
+
+module "staging_lambda" {
+  source = "./modules/transferrer_pipe"
+  environment = "staging"
+  queue_visibility_timeout = local.queue_visibility_timeout
+  lambda_zip = data.archive_file.lambda_zip
+}
+
+module "production_lambda" {
+  source = "./modules/transferrer_pipe"
+  environment = "production"
+  queue_visibility_timeout = local.queue_visibility_timeout
+  lambda_zip = data.archive_file.lambda_zip
+}

terraform/modules/notification_queue/main.tf

@@ -0,0 +1,22 @@
+
+module "transfer_shoots_topic" {
+  source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.1"
+  name = "transfer-shoots-${var.environment}"
+}
+
+module "dlq_alarm_topic" {
+  source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.1"
+  name = "transfer-shoots-alarm-${var.environment}"
+}
+
+module "input_queue" {
+  source = "github.com/wellcomecollection/terraform-aws-sqs//queue?ref=v1.2.1"
+
+  queue_name = "transfer-shoots-${var.environment}"
+
+  topic_arns                 = [module.transfer_shoots_topic.arn]
+  visibility_timeout_seconds = var.queue_visibility_timeout
+  max_receive_count          = 1
+  message_retention_seconds = 1200
+  alarm_topic_arn           = module.dlq_alarm_topic.arn
+}

terraform/modules/notification_queue/outputs.tf

@@ -0,0 +1,3 @@
+output "queue_arn" {
+  value = module.input_queue.arn
+}

terraform/modules/notification_queue/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/notification_queue/variables.tf

@@ -0,0 +1,11 @@
+variable "queue_visibility_timeout" {
+  type = number
+}
+
+variable "environment" {
+  type = string
+  validation {
+    condition = contains(["staging", "production"], var.environment)
+    error_message = "environment must be one of staging or production"
+  }
+}

terraform/modules/sqs_lambda_trigger/main.tf

@@ -0,0 +1,33 @@
+
+data "aws_iam_policy_document" "allow_sqs_pull" {
+    statement {
+      actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes"
+    ]
+    resources = [
+      var.queue_arn
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "allow_sqs_pull" {
+  name   = "${var.trigger_name}-pull-from-queue"
+  role   = var.role_name
+  policy = data.aws_iam_policy_document.allow_sqs_pull.json
+}
+
+resource "aws_lambda_event_source_mapping" "lambda_trigger" {
+  event_source_arn = var.queue_arn
+  enabled          = true
+  function_name    = var.function_name
+  batch_size       = var.batch_size
+}
+
+resource "aws_lambda_permission" "allow_lambda_sqs_trigger" {
+  action        = "lambda:InvokeFunction"
+  function_name    = var.function_name
+  principal     = "sqs.amazonaws.com"
+  source_arn = var.queue_arn
+}

terraform/modules/sqs_lambda_trigger/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/sqs_lambda_trigger/variables.tf

@@ -0,0 +1,19 @@
+variable "queue_arn" {
+  type = string
+}
+
+variable "function_name" {
+  type = string
+}
+
+variable "role_name" {
+    type = string
+}
+
+variable "trigger_name" {
+  type = string
+}
+variable "batch_size" {
+  type = number
+  default = 1
+}

terraform/modules/transferrer_lambda/main.tf

@@ -0,0 +1,71 @@
+locals {
+  lambda_name = "editorial-photography-transfer-${var.environment}"
+  lambda_timeout = 120 //two minutes
+  buckets = tomap(
+    {
+      staging = "wellcomecollection-archivematica-staging-transfer-source",
+      production = "wellcomecollection-archivematica-transfer-source"
+    }
+  )
+  target_bucket = lookup(local.buckets, var.environment)
+
+}
+
+
+module "transfer_lambda" {
+  source = "git@github.com:wellcomecollection/terraform-aws-lambda?ref=v1.2.0"
+
+  name    = local.lambda_name
+  runtime = "python3.12"
+  handler = "lambda_function.lambda_handler"
+
+  filename    = var.lambda_zip.output_path
+  memory_size = 512
+  timeout     = local.lambda_timeout
+
+  environment = {
+    variables = {
+      ACCESSION_NUMBER = "2754"
+      TARGET_BUCKET = local.target_bucket
+    }
+  }
+  source_code_hash = var.lambda_zip.output_base64sha256
+}
+
+resource "aws_iam_role_policy" "write_to_archivematica_transfer_source" {
+  role = module.transfer_lambda.lambda_role.name
+  name = "write_to_archivematica_transfer_source-${var.environment}"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+            "Effect": "Allow",
+            "Action": "s3:PutObject",
+            "Resource": "arn:aws:s3:::${local.target_bucket}/*"
+        },
+    ]
+  }
+  )
+}
+
+resource "aws_iam_role_policy" "read_from_editorial_photography" {
+  role = module.transfer_lambda.lambda_role.name
+  name = "read_from_editorial_photography-${var.environment}"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+            "Effect" =  "Allow",
+            "Action" =  [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource" = [
+              "arn:aws:s3:::wellcomecollection-editorial-photography",
+              "arn:aws:s3:::wellcomecollection-editorial-photography/*"
+            ],
+        },
+    ]
+  })
+
+}

terraform/modules/transferrer_lambda/outputs.tf

@@ -0,0 +1,7 @@
+output "lambda" {
+  value = module.transfer_lambda.lambda
+}
+
+output "role" {
+  value = module.transfer_lambda.lambda_role
+}

terraform/modules/transferrer_lambda/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/transferrer_lambda/variables.tf

@@ -0,0 +1,16 @@
+variable "environment" {
+  type = string
+  validation {
+    condition = contains(["staging", "production"], var.environment)
+    error_message = "environment must be one of staging or production"
+  }
+}
+
+variable "lambda_zip" {
+  type = object(
+    {
+      output_path = string,
+      output_base64sha256 = string
+    }
+  )
+}

terraform/modules/transferrer_pipe/main.tf

@@ -0,0 +1,20 @@
+
+module "transfer_lambda" {
+  source = "../transferrer_lambda"
+  environment = var.environment
+  lambda_zip = var.lambda_zip
+}
+
+module "input_queue" {
+  source = "../notification_queue"
+  environment = var.environment
+  queue_visibility_timeout = var.queue_visibility_timeout
+}
+
+module "trigger" {
+  source = "../sqs_lambda_trigger"
+  queue_arn = module.input_queue.queue_arn
+  function_name = module.transfer_lambda.lambda.function_name
+  role_name = module.transfer_lambda.role.name
+  trigger_name = "editorial-photography-${var.environment}"
+}

terraform/modules/transferrer_pipe/provider.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}

terraform/modules/transferrer_pipe/variables.tf

@@ -0,0 +1,20 @@
+variable "environment" {
+  type = string
+  validation {
+    condition = contains(["staging", "production"], var.environment)
+    error_message = "environment must be one of staging or production"
+  }
+}
+
+variable "queue_visibility_timeout" {
+  type = number
+}
+
+variable "lambda_zip" {
+  type = object(
+    {
+      output_path = string,
+      output_base64sha256 = string
+    }
+  )
+}

terraform/provider.tf

@@ -0,0 +1,8 @@
+
+provider "aws" {
+  region = "eu-west-1"
+
+  assume_role {
+    role_arn = "arn:aws:iam::404315009621:role/digitisation-developer"
+  }
+}