allow waiting after each deletion (#68)

* allow to wait a fixed amount of seconds after issuing a delete

This may be useful to throttle deletions against a slow backend,
to avoid big batch clean ups from slowing down other requests.

* don't sleep during dry runs

* add debug output before sleep to be able to detect if deletion is slow

README.rst

@@ -97,6 +97,8 @@ Available command line options:
  Run only once and exit. This is useful if you run the Kubernetes Janitor as a ``CronJob``.
 ``--interval``
  Loop interval (default: 30s). This option only makes sense when the ``--once`` flag is not set.
+``--wait-after-delete``
+ How long to wait after issuing a delete (default: 0s). This option does not take effect for dry runs.
 ``--include-resources``
  Include resources for clean up (default: all resources), can also be configured via environment variable ``INCLUDE_RESOURCES``. This option can be used if you want to clean up only certain resource types, e.g. only ``deployments``.
 ``--exclude-resources``

kube_janitor/cmd.py

@@ -32,6 +32,12 @@ def get_parser():
  parser.add_argument(
  "--interval", type=int, help="Loop interval (default: 30s)", default=30
  )
+ parser.add_argument(
+ "--wait-after-delete",
+ type=int,
+ help="Wait time after issuing a delete (default: 0s)",
+ default=0,
+ )
  parser.add_argument(
  "--delete-notification",
  type=int,

kube_janitor/janitor.py

@@ -1,5 +1,6 @@
 import datetime
 import logging
+import time
 from collections import Counter
 from typing import Any
 from typing import Callable
@@ -148,7 +149,7 @@ def create_event(resource, message: str, reason: str, dry_run: bool):
  logger.error(f"Could not create event {event.obj}: {e}")
 
 
-def delete(resource, dry_run: bool):
+def delete(resource, wait_after_delete: int, dry_run: bool):
  if dry_run:
  logger.info(
  f"**DRY-RUN**: would delete {resource.kind} {resource.namespace}/{resource.name}"
@@ -166,6 +167,9 @@ def delete(resource, dry_run: bool):
  logger.error(
  f"Could not delete {resource.kind} {resource.namespace}/{resource.name}: {e}"
  )
+ if wait_after_delete > 0:
+ logger.debug(f"Waiting {wait_after_delete}s after deletion")
+ time.sleep(wait_after_delete)
 
 
 def handle_resource_on_ttl(
@@ -175,6 +179,7 @@ def handle_resource_on_ttl(
  deployment_time_annotation: Optional[str] = None,
  resource_context_hook: Optional[Callable[[APIObject, dict], Dict[str, Any]]] = None,
  cache: Dict[str, Any] = None,
+ wait_after_delete: int = 0,
  dry_run: bool = False,
 ):
  counter = {"resources-processed": 1}
@@ -216,7 +221,9 @@ def handle_resource_on_ttl(
  create_event(
  resource, message, "TimeToLiveExpired", dry_run=dry_run
  )
- delete(resource, dry_run=dry_run)
+ delete(
+ resource, wait_after_delete=wait_after_delete, dry_run=dry_run
+ )
  counter[f"{resource.endpoint}-deleted"] = 1
  elif delete_notification:
  expiry_time = deployment_time + datetime.timedelta(
@@ -233,7 +240,9 @@ def handle_resource_on_ttl(
  return counter
 
 
-def handle_resource_on_expiry(resource, rules, delete_notification: int, dry_run: bool):
+def handle_resource_on_expiry(
+ resource, rules, delete_notification: int, wait_after_delete: int, dry_run: bool
+):
  counter = {}
 
  expiry = resource.annotations.get(EXPIRY_ANNOTATION)
@@ -252,7 +261,7 @@ def handle_resource_on_expiry(resource, rules, delete_notification: int, dry_run
  message = f"{resource.kind} {resource.name} expired on {expiry} and will be deleted ({reason})"
  logger.info(message)
  create_event(resource, message, "ExpiryTimeReached", dry_run=dry_run)
- delete(resource, dry_run=dry_run)
+ delete(resource, wait_after_delete=wait_after_delete, dry_run=dry_run)
  counter[f"{resource.endpoint}-deleted"] = 1
  else:
  logging.debug(
@@ -280,6 +289,7 @@ def clean_up(
  delete_notification: int,
  deployment_time_annotation: Optional[str] = None,
  resource_context_hook: Optional[Callable[[APIObject, dict], Dict[str, Any]]] = None,
+ wait_after_delete: int = 0,
  dry_run: bool = False,
 ):
 
@@ -302,12 +312,13 @@ def clean_up(
  deployment_time_annotation,
  resource_context_hook,
  cache,
+ wait_after_delete,
  dry_run,
  )
  )
  counter.update(
  handle_resource_on_expiry(
- namespace, rules, delete_notification, dry_run
+ namespace, rules, delete_notification, wait_after_delete, dry_run
  )
  )
  else:
@@ -352,11 +363,14 @@ def clean_up(
  deployment_time_annotation,
  resource_context_hook,
  cache,
+ wait_after_delete,
  dry_run,
  )
  )
  counter.update(
- handle_resource_on_expiry(resource, rules, delete_notification, dry_run)
+ handle_resource_on_expiry(
+ resource, rules, delete_notification, wait_after_delete, dry_run
+ )
  )
  stats = ", ".join([f"{k}={v}" for k, v in counter.items()])
  logger.info(f"Clean up run completed: {stats}")

kube_janitor/main.py

@@ -46,6 +46,7 @@ def main(args=None):
  args.delete_notification,
  args.deployment_time_annotation,
  args.resource_context_hook,
+ args.wait_after_delete,
  args.dry_run,
  )
 
@@ -61,6 +62,7 @@ def run_loop(
  delete_notification,
  deployment_time_annotation: Optional[str],
  resource_context_hook: Optional[Callable],
+ wait_after_delete: int,
  dry_run: bool,
 ):
  handler = shutdown.GracefulShutdown()
@@ -77,6 +79,7 @@ def run_loop(
  delete_notification=delete_notification,
  deployment_time_annotation=deployment_time_annotation,
  resource_context_hook=resource_context_hook,
+ wait_after_delete=wait_after_delete,
  dry_run=dry_run,
  )
  except Exception as e:

tests/test_clean_up.py

@@ -37,11 +37,24 @@ def test_delete_namespace(caplog):
  caplog.set_level(logging.INFO)
  mock_api = MagicMock()
  foo_ns = Namespace(mock_api, {"metadata": {"name": "foo"}})
- delete(foo_ns, dry_run=False)
+ delete(foo_ns, wait_after_delete=0, dry_run=False)
  assert "Deleting Namespace foo.." in caplog.messages
  mock_api.delete.assert_called_once()
 
 
+def test_wait_after_delete(monkeypatch):
+ mock_sleep = MagicMock()
+ monkeypatch.setattr("time.sleep", mock_sleep)
+ mock_api = MagicMock()
+ foo_ns = Namespace(mock_api, {"metadata": {"name": "foo"}})
+
+ delete(foo_ns, wait_after_delete=123, dry_run=True)
+ assert not mock_sleep.called
+
+ delete(foo_ns, wait_after_delete=123, dry_run=False)
+ mock_sleep.assert_called_once_with(123)
+
+
 def test_handle_resource_no_ttl():
  resource = Namespace(None, {"metadata": {"name": "foo"}})
  counter = handle_resource_on_ttl(
@@ -52,7 +65,9 @@ def test_handle_resource_no_ttl():
 
 def test_handle_resource_no_expiry():
  resource = Namespace(None, {"metadata": {"name": "foo"}})
- counter = handle_resource_on_expiry(resource, [], None, dry_run=True)
+ counter = handle_resource_on_expiry(
+ resource, [], None, wait_after_delete=0, dry_run=True
+ )
  assert counter == {}
 
 
@@ -213,7 +228,9 @@ def test_handle_resource_expiry_annotation():
  }
  },
  )
- counter = handle_resource_on_expiry(resource, [], None, dry_run=True)
+ counter = handle_resource_on_expiry(
+ resource, [], None, wait_after_delete=0, dry_run=True
+ )
  assert counter == {"namespaces-with-expiry": 1}
 
 
@@ -250,7 +267,9 @@ def test_handle_resource_expiry_expired():
  }
  },
  )
- counter = handle_resource_on_expiry(resource, [], None, dry_run=True)
+ counter = handle_resource_on_expiry(
+ resource, [], None, wait_after_delete=0, dry_run=True
+ )
  assert counter == {"namespaces-with-expiry": 1, "namespaces-deleted": 1}
 
 

tests/test_delete_notification.py

@@ -280,7 +280,9 @@ def test_handle_resource_expiry_annotation_with_notification_not_triggered(
  },
  )
  delete_notification = 180
- handle_resource_on_expiry(resource, [], delete_notification, dry_run=True)
+ handle_resource_on_expiry(
+ resource, [], delete_notification, wait_after_delete=0, dry_run=True
+ )
  mocked_add_notification_flag.assert_not_called()
 
  @unittest.mock.patch(
@@ -308,7 +310,9 @@ def test_handle_resource_expiry_annotation_with_notification_triggered(
  },
  )
  delete_notification = 180
- handle_resource_on_expiry(resource, [], delete_notification, dry_run=True)
+ handle_resource_on_expiry(
+ resource, [], delete_notification, wait_after_delete=0, dry_run=True
+ )
  mocked_add_notification_flag.assert_called()
 
  @unittest.mock.patch(
@@ -336,7 +340,9 @@ def test_handle_resource_expiry_annotation_notification_event(
  },
  )
  delete_notification = 180
- handle_resource_on_expiry(resource, [], delete_notification, dry_run=True)
+ handle_resource_on_expiry(
+ resource, [], delete_notification, wait_after_delete=0, dry_run=True
+ )
 
  expire = datetime.datetime.strptime(
  "2019-03-11T11:15:00Z", "%Y-%m-%dT%H:%M:%SZ"