Merge branch 'main' into bb/remove-boilerplate

acceptance/run.sh

@@ -154,7 +154,7 @@ tests() {
  "$PLAN_NAME" \
  "$INSTANCE" \
  -c "{\"domains\": \"$DOMAIN_0, $DOMAIN_1\", \"forward_cookies\": \"cookieone, cookietwo\", \"forward_headers\": \"x-one-header,x-two-header\", \"error_responses\": {\"404\": \"/errors/404.html\"}}"
- elif [[ "${PLAN_NAME}" == "domain-with-cdn-dedicated-wafP" ]]; then
+ elif [[ "${PLAN_NAME}" == "domain-with-cdn-dedicated-waf" ]]; then
  echo "Creating the service instance"
  cf create-service \
  "$SERVICE_NAME" \

broker/lib/cdn.py

@@ -1,14 +1,14 @@
-from broker.models import ServiceInstanceTypes
+from broker.models import (
+ CDNServiceInstance,
+ CDNDedicatedWAFServiceInstance,
+)
 
 
 def is_cdn_instance(service_instance) -> bool:
- return service_instance.instance_type in [
- ServiceInstanceTypes.CDN.value,
- ServiceInstanceTypes.CDN_DEDICATED_WAF.value,
- ]
+ return isinstance(service_instance, CDNServiceInstance) or isinstance(
+ service_instance, CDNDedicatedWAFServiceInstance
+ )
 
 
 def is_cdn_dedicated_waf_instance(service_instance) -> bool:
- return (
- service_instance.instance_type == ServiceInstanceTypes.CDN_DEDICATED_WAF.value
- )
+ return isinstance(service_instance, CDNDedicatedWAFServiceInstance)

broker/pipelines/cdn_dedicated_waf.py

@@ -94,10 +94,13 @@ def queue_all_cdn_dedicated_waf_update_tasks_for_operation(
  .then(route53.create_ALIAS_records, operation_id, **correlation)
  .then(route53.wait_for_changes, operation_id, **correlation)
  .then(iam.delete_previous_server_certificate, operation_id, **correlation)
+ .then(sns.create_notification_topic, operation_id, **correlation)
  .then(route53.create_new_health_checks, operation_id, **correlation)
  .then(shield.update_associated_health_check, operation_id, **correlation)
  .then(route53.delete_unused_health_checks, operation_id, **correlation)
- .then(cloudwatch.update_health_check_alarms, operation_id, **correlation)
+ .then(cloudwatch.delete_health_check_alarms, operation_id, **correlation)
+ .then(cloudwatch.create_health_check_alarms, operation_id, **correlation)
+ .then(cloudwatch.create_ddos_detected_alarm, operation_id, **correlation)
  .then(update_operations.update_complete, operation_id, **correlation)
  )
  huey.enqueue(task_pipeline)

broker/tasks/cloudwatch.py

@@ -142,6 +142,12 @@ def create_ddos_detected_alarm(operation_id: int, *, operation, db, **kwargs):
  f"Could not find sns_notification_topic_arn for instance {service_instance.id}"
  )
 
+ if service_instance.ddos_detected_cloudwatch_alarm_name:
+ logger.info(
+ f"DDoS alarm name {service_instance.ddos_detected_cloudwatch_alarm_name} already exists"
+ )
+ return
+
  ddos_detected_alarm_name = generate_ddos_alarm_name(service_instance.id)
  _create_cloudwatch_alarm(
  generate_ddos_alarm_name(service_instance.id),

broker/tasks/sns.py

@@ -16,6 +16,10 @@ def create_notification_topic(operation_id: int, *, operation, db, **kwargs):
  service_instance = operation.service_instance
 
  sns_kwargs = {}
+ if service_instance.sns_notification_topic_arn:
+ logger.info(f"Topic already exists for instance {service_instance.id}")
+ return
+
  if service_instance.tags:
  sns_kwargs["Tags"] = service_instance.tags
 

tests/integration/api/test_server_setup.py

@@ -0,0 +1,7 @@
+from broker.models import DedicatedALBListener
+
+
+def test_server_runs(client):
+ client.get("/ping")
+ assert client.response.status_code == 200
+ assert client.response.body == "PONG"

tests/integration/cdn_dedicated_waf/test_cdn_dedicated_waf_provisioning.py

@@ -62,8 +62,10 @@
  subtest_updates_associated_health_check_no_change,
  subtest_update_creates_new_health_checks,
  subtest_update_deletes_unused_health_checks,
- subtest_updates_health_check_alarms,
- subtest_updates_health_check_alarms_no_change,
+ subtest_update_deletes_health_check_alarms,
+ subtest_update_creates_health_check_alarms,
+ subtest_update_does_not_create_sns_notification_topic,
+ subtest_update_does_not_create_ddos_cloudwatch_alarm,
 )
 
 
@@ -195,6 +197,7 @@ def test_provision_happy_path(
  wafv2,
  shield,
  cloudwatch_commercial,
+ sns_commercial,
  instance_model,
  )
  subtest_update_same_domains(
@@ -206,6 +209,7 @@ def test_provision_happy_path(
  wafv2,
  shield,
  cloudwatch_commercial,
+ sns_commercial,
  instance_model,
  )
 
@@ -221,6 +225,7 @@ def subtest_update_happy_path(
  wafv2,
  shield,
  cloudwatch_commercial,
+ sns_commercial,
  instance_model,
 ):
  operation_id = subtest_update_creates_update_operation(client, dns, instance_model)
@@ -242,6 +247,9 @@ def subtest_update_happy_path(
  subtest_update_updates_ALIAS_records(tasks, route53, instance_model)
  subtest_waits_for_dns_changes(tasks, route53, instance_model)
  subtest_update_removes_certificate_from_iam(tasks, iam_commercial, instance_model)
+ subtest_update_does_not_create_sns_notification_topic(
+ tasks, sns_commercial, instance_model
+ )
  subtest_update_creates_new_health_checks(tasks, route53, instance_model)
  check_last_operation_description(
  client, "4321", operation_id, "Creating new health checks"
@@ -254,12 +262,32 @@ def subtest_update_happy_path(
  check_last_operation_description(
  client, "4321", operation_id, "Deleting unused health checks"
  )
- subtest_updates_health_check_alarms(tasks, cloudwatch_commercial, instance_model)
+ subtest_update_deletes_health_check_alarms(
+ tasks,
+ cloudwatch_commercial,
+ instance_model,
+ ["example.com ID", "foo.com ID"],
+ )
  check_last_operation_description(
  client,
  "4321",
  operation_id,
- "Updating Cloudwatch alarms for Route53 health checks",
+ "Deleting Cloudwatch alarms for Route53 health checks",
+ )
+ subtest_update_creates_health_check_alarms(
+ tasks, cloudwatch_commercial, instance_model
+ )
+ check_last_operation_description(
+ client,
+ "4321",
+ operation_id,
+ "Creating Cloudwatch alarms for Route53 health checks",
+ )
+ subtest_update_does_not_create_ddos_cloudwatch_alarm(
+ tasks, cloudwatch_commercial, instance_model
+ )
+ check_last_operation_description(
+ client, "4321", operation_id, "Creating DDoS detection alarm"
  )
  subtest_update_marks_update_complete(tasks, instance_model)
 
@@ -273,6 +301,7 @@ def subtest_update_same_domains(
  wafv2,
  shield,
  cloudwatch_commercial,
+ sns_commercial,
  instance_model,
 ):
  subtest_update_same_domains_creates_update_operation(client, dns, instance_model)
@@ -295,9 +324,22 @@ def subtest_update_same_domains(
  subtest_update_same_domains_does_not_delete_server_certificate(
  tasks, instance_model
  )
+ subtest_update_does_not_create_sns_notification_topic(
+ tasks, sns_commercial, instance_model
+ )
  subtest_updates_health_checks_do_not_change(tasks, route53, instance_model)
  subtest_updates_associated_health_check_no_change(tasks, shield, instance_model)
- subtest_updates_health_check_alarms_no_change(
+ subtest_updates_health_checks_do_not_change(tasks, route53, instance_model)
+ subtest_update_deletes_health_check_alarms(
+ tasks,
+ cloudwatch_commercial,
+ instance_model,
+ ["bar.com ID", "foo.com ID"],
+ )
+ subtest_update_creates_health_check_alarms(
+ tasks, cloudwatch_commercial, instance_model
+ )
+ subtest_update_does_not_create_ddos_cloudwatch_alarm(
  tasks, cloudwatch_commercial, instance_model
  )
  subtest_update_marks_update_complete(tasks, instance_model)

tests/integration/cdn_dedicated_waf/update.py

@@ -138,54 +138,96 @@ def subtest_updates_associated_health_check_no_change(tasks, shield, instance_mo
  assert service_instance.shield_associated_health_check == check_pre_update
 
 
-def subtest_updates_health_check_alarms(
+def subtest_update_deletes_health_check_alarms(
  tasks,
  cloudwatch_commercial,
  instance_model,
+ expect_delete_health_check_ids,
  service_instance_id="4321",
 ):
+ expect_delete_alarm_names = [
+ _get_alarm_name(expect_delete_health_check_id)
+ for expect_delete_health_check_id in expect_delete_health_check_ids
+ ]
+ cloudwatch_commercial.expect_delete_alarms(expect_delete_alarm_names)
+
+ tasks.run_queued_tasks_and_enqueue_dependents()
+
  db.session.expunge_all()
  service_instance = db.session.get(instance_model, service_instance_id)
 
- expected_health_check_alarms = []
- expect_delete_health_check_id = "example.com ID"
- expect_delete_alarm_names = [_get_alarm_name(expect_delete_health_check_id)]
- expect_create_health_check_id = "bar.com ID"
- expected_health_check_alarms = [
- {
- "health_check_id": "foo.com ID",
- "alarm_name": _get_alarm_name("foo.com ID"),
- },
- {
- "health_check_id": expect_create_health_check_id,
- "alarm_name": _get_alarm_name(expect_create_health_check_id),
- },
- ]
+ assert service_instance.cloudwatch_health_check_alarms == []
 
- cloudwatch_commercial.expect_delete_alarms(expect_delete_alarm_names)
- cloudwatch_commercial.expect_put_metric_alarm(
- expect_create_health_check_id,
- _get_alarm_name(expect_create_health_check_id),
- service_instance,
- )
- cloudwatch_commercial.expect_describe_alarms(
- _get_alarm_name(expect_create_health_check_id),
- [{"AlarmArn": f"{expect_create_health_check_id} ARN"}],
- )
+ cloudwatch_commercial.assert_no_pending_responses()
+
+
+def subtest_update_creates_health_check_alarms(
+ tasks,
+ cloudwatch_commercial,
+ instance_model,
+ service_instance_id="4321",
+):
+ db.session.expunge_all()
+ service_instance = db.session.get(instance_model, service_instance_id)
+
+ expect_create_health_check_ids = ["bar.com ID", "foo.com ID"]
+ for expect_create_health_check_id in expect_create_health_check_ids:
+ cloudwatch_commercial.expect_put_metric_alarm(
+ expect_create_health_check_id,
+ _get_alarm_name(expect_create_health_check_id),
+ service_instance,
+ )
+ cloudwatch_commercial.expect_describe_alarms(
+ _get_alarm_name(expect_create_health_check_id),
+ [{"AlarmArn": f"{expect_create_health_check_id} ARN"}],
+ )
 
  tasks.run_queued_tasks_and_enqueue_dependents()
 
  db.session.expunge_all()
  service_instance = db.session.get(instance_model, service_instance_id)
 
- assert (
- service_instance.cloudwatch_health_check_alarms == expected_health_check_alarms
+ assert sorted(
+ service_instance.cloudwatch_health_check_alarms,
+ key=lambda alarm: alarm["health_check_id"],
+ ) == sorted(
+ [
+ {
+ "health_check_id": "foo.com ID",
+ "alarm_name": _get_alarm_name("foo.com ID"),
+ },
+ {
+ "health_check_id": "bar.com ID",
+ "alarm_name": _get_alarm_name("bar.com ID"),
+ },
+ ],
+ key=lambda alarm: alarm["health_check_id"],
  )
 
  cloudwatch_commercial.assert_no_pending_responses()
 
 
-def subtest_updates_health_check_alarms_no_change(
+def subtest_update_does_not_create_sns_notification_topic(
+ tasks,
+ sns_commercial,
+ instance_model,
+ service_instance_id="4321",
+):
+ db.session.expunge_all()
+ service_instance = db.session.get(instance_model, service_instance_id)
+
+ assert service_instance.sns_notification_topic_arn
+
+ tasks.run_queued_tasks_and_enqueue_dependents()
+ sns_commercial.assert_no_pending_responses()
+
+ db.session.expunge_all()
+ service_instance = db.session.get(instance_model, service_instance_id)
+
+ assert service_instance.sns_notification_topic_arn
+
+
+def subtest_update_does_not_create_ddos_cloudwatch_alarm(
  tasks,
  cloudwatch_commercial,
  instance_model,
@@ -194,14 +236,12 @@ def subtest_updates_health_check_alarms_no_change(
  db.session.expunge_all()
  service_instance = db.session.get(instance_model, service_instance_id)
 
- health_check_alarms_pre_update = service_instance.cloudwatch_health_check_alarms
+ assert service_instance.ddos_detected_cloudwatch_alarm_name
 
  tasks.run_queued_tasks_and_enqueue_dependents()
  cloudwatch_commercial.assert_no_pending_responses()
 
  db.session.expunge_all()
  service_instance = db.session.get(instance_model, service_instance_id)
- assert (
- service_instance.cloudwatch_health_check_alarms
- == health_check_alarms_pre_update
- )
+
+ assert service_instance.ddos_detected_cloudwatch_alarm_name

tests/integration/test_canceling_ops.py → tests/integration/huey/test_canceling_ops.py

@@ -1,12 +1,11 @@
-import json
-from datetime import date, datetime
+from datetime import datetime
 
 import pytest # noqa F401
 
 from huey.exceptions import CancelExecution
 
-from broker.extensions import config, db
-from broker.models import Challenge, Operation, CDNServiceInstance
+from broker.extensions import db
+from broker.models import Operation
 from tests.lib.factories import (
  OperationFactory,
  CDNServiceInstanceFactory,

tests/integration/test_failing_tasks.py → tests/integration/huey/test_failing_tasks.py

@@ -2,9 +2,8 @@
 import pytest
 
 from huey.exceptions import TaskException
-from broker.extensions import config, db
+from broker.extensions import db
 from broker.tasks.huey import huey
-from broker.tasks.letsencrypt import initiate_challenges
 from broker.models import Operation
 
 from tests.lib.factories import OperationFactory

tests/integration/test_server_setup.py → tests/integration/models/test_dedicated_alb_listener.py

@@ -1,14 +1,6 @@
-import pytest
-
 from broker.models import DedicatedALBListener
 
 
-def test_server_runs(client):
- client.get("/ping")
- assert client.response.status_code == 200
- assert client.response.body == "PONG"
-
-
 def test_load_albs_on_startup(clean_db):
  listeners = DedicatedALBListener.query.all()
  assert len(listeners) == 0