Merge pull request #3 from hjacobs/rules

Support generic rules to define TTL for arbitrary resources

Pipfile

@@ -6,6 +6,7 @@ name = "pypi"
 [packages]
 pykube = "*"
 pytz = "*"
+jmespath = "*"
 
 [dev-packages]
 flake8 = "*"

README.rst

@@ -49,7 +49,7 @@ You should see the ``temp-nginx`` deployment being deleted after 5 minutes.
 Configuration
 =============
 
-The janitor is configured via command line args, environment variables and Kubernetes annotations.
+The janitor is configured via command line args, environment variables, Kubernetes annotations, and an optional YAML rules file.
 
 Kubernetes annotations:
 
@@ -75,6 +75,53 @@ Available command line options:
  Include namespaces for clean up (default: all namespaces), can also be configured via environment variable ``INCLUDE_NAMESPACES``
 ``--exclude-namespaces``
  Exclude namespaces from clean up (default: kube-system), can also be configured via environment variable ``EXCLUDE_NAMESPACES``
+``--rules-file``
+ Optional: filename pointing to a YAML file with a list of rules to apply TTL values to arbitrary Kubernetes objects, e.g. to delete all deployments without a certain label automatically after N days. See Rules File configuration section below.
+
+
+Rules File
+==========
+
+When using the ``--rules-file`` option, the path needs to point to a valid YAML file with the following format:
+
+.. code-block:: yaml
+
+ rules:
+ # remove deployments and statefulsets without a label "application"
+ - id: require-application-label
+ resources:
+ - deployments
+ - statefulsets
+ jmespath: "!(spec.template.metadata.labels.application)"
+ ttl: 4d
+ # delete all deployments with a name starting with "pr-*"
+ - id: temporary-pr-deployments
+ resources:
+ - deployments
+ jmespath: "starts_with(metadata.name, 'pr-')"
+ ttl: 4h
+
+The first matching rule will define the TTL (``ttl`` field). Kubernetes objects with a ``janitor/ttl`` annotation will not be matched against any rule.
+
+A rule matches for a given Kubernetes object if all of the following criteria is true:
+
+* the object has no ``janitor/ttl`` annotation (otherwise the TTL value from the annotation is applied)
+* the object's type is included in the ``resources`` list of the rule or the special value ``*`` is part of the ``resources`` list (similar to Kubernetes RBAC)
+* the JMESPath_ evaluates to a truth-like value (boolean ``true``, non-empty list, non-empty object, or non-empty string)
+
+The first matching rule will define the TTL for the object (as if the object would have a ``janitor/ttl`` annotation with the same value).
+
+Each rule has the following attributes:
+
+``id``
+ Some string identifying the rule (e.g. for log output), must be lowercase and match the regex ``^[a-z][a-z0-9-]*$``. The ID has no special meaning and is only used to refer to the rule in log output/statistics.
+``resources``
+ List of resources (e.g. ``deployments``, ``namespaces``, ..) this rule should be applied to. The special value ``*`` will match all resource types.
+``jmespath``
+ JMESPath_ expression to evaluate on the resource object. The rule will only match if the expression evaluates to true. The expression will get the Kubernetes object as input.
+ The expression ``metadata.labels.foo`` would evaluate to true if the object has the label ``foo`` and it has a non-empty string as value.
+``ttl``
+ TTL value (e.g. ``15m``) to apply to the object if the rule matches.
 
 
 Contributing
@@ -123,3 +170,4 @@ along with this program. If not, see http://www.gnu.org/licenses/.
 .. _Minikube: https://github.com/kubernetes/minikube
 .. _ping try_except_ on Twitter: https://twitter.com/try_except_
 .. _issues labeled with "help wanted": https://github.com/hjacobs/kube-janitor/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
+.. _JMESPath: http://jmespath.org/

deploy/deployment.yaml

@@ -20,14 +20,15 @@ spec:
  containers:
  - name: janitor
  # see https://github.com/hjacobs/kube-janitor/releases
- image: hjacobs/kube-janitor:0.1
+ image: hjacobs/kube-janitor:0.1-10-gce8e1da
  args:
  # dry run by default, remove to perform clean up
  - --dry-run
- # uncomment to have verbose logging
- # - --debug
+ # comment out to have less verbose logging
+ - --debug
  # run every minute
  - --interval=60
+ - --rules-file=/config/rules.yaml
  resources:
  limits:
  memory: 100Mi
@@ -38,3 +39,10 @@ spec:
  readOnlyRootFilesystem: true
  runAsNonRoot: true
  runAsUser: 1000
+ volumeMounts:
+ - name: config-volume
+ mountPath: /config
+ volumes:
+ - name: config-volume
+ configMap:
+ name: kube-janitor

deploy/rules-config.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: kube-janitor
+data:
+ rules.yaml: |-
+ # example rules configuration to set TTL for arbitrary objects
+ # see https://github.com/hjacobs/kube-janitor for details
+ rules:
+ - id: require-application-label
+ # remove deployments and statefulsets without a label "application"
+ resources:
+ # resources are prefixed with "XXX" to make sure they are not active by accident
+ # modify the rule as needed and remove the "XXX" prefix to activate
+ - XXXdeployments
+ - XXXstatefulsets
+ # see http://jmespath.org/specification.html
+ jmespath: "!(spec.template.metadata.labels.application)"
+ ttl: 4d
+ - id: temporary-pr-namespaces
+ # delete all namespaces with a name starting with "pr-*"
+ resources:
+ # resources are prefixed with "XXX" to make sure they are not active by accident
+ # modify the rule as needed and remove the "XXX" prefix to activate
+ - XXXnamespaces
+ # this uses JMESPath's built-in "starts_with" function
+ # see http://jmespath.org/specification.html#starts-with
+ jmespath: "starts_with(metadata.name, 'pr-')"
+ ttl: 4h

kube_janitor/cmd.py

@@ -21,4 +21,6 @@ def get_parser():
  default=os.getenv('INCLUDE_NAMESPACES', 'all'))
  parser.add_argument('--exclude-namespaces', help=f'Exclude namespaces from clean up (default: {DEFAULT_EXCLUDE_NAMESPACES})',
  default=os.getenv('EXCLUDE_NAMESPACES', DEFAULT_EXCLUDE_NAMESPACES))
+ parser.add_argument('--rules-file', help='Load TTL rules from given file path',
+ default=os.getenv('RULES_FILE'))
  return parser

kube_janitor/janitor.py

@@ -2,6 +2,8 @@
 import logging
 import pykube
 
+from collections import Counter
+
 from .helper import parse_ttl
 from .resources import get_namespaced_resource_types
 from pykube import Namespace
@@ -48,31 +50,48 @@ def delete(resource, dry_run: bool):
  logger.error(f'Could not delete {resource.kind} {resource.namespace}/{resource.name}: {e}')
 
 
-def handle_resource(resource, dry_run: bool):
+def handle_resource(resource, rules, dry_run: bool):
+ counter = {'resources-processed': 1}
+
  ttl = resource.annotations.get(TTL_ANNOTATION)
+ if not ttl:
+ for rule in rules:
+ if rule.matches(resource):
+ logger.debug(f'Rule {rule.id} applies {rule.ttl} TTL to {resource.kind} {resource.namespace}/{resource.name}')
+ ttl = rule.ttl
+ counter[f'rule-{rule.id}-matches'] = 1
+ # first rule which matches
+ break
  if ttl:
  try:
  ttl_seconds = parse_ttl(ttl)
  except ValueError as e:
  logger.info(f'Ignoring invalid TTL on {resource.kind} {resource.name}: {e}')
  else:
+ counter[f'{resource.endpoint}-with-ttl'] = 1
  age = get_age(resource)
  logger.debug(f'{resource.kind} {resource.name} with TTL of {ttl} is {age} old')
  if age.total_seconds() > ttl_seconds:
  logger.info(f'{resource.kind} {resource.name} with TTL of {ttl} is {age} old and will be deleted')
  delete(resource, dry_run=dry_run)
+ counter['{resource.endpoint}-deleted'] = 1
+
+ return counter
 
 
 def clean_up(api,
  include_resources: frozenset,
  exclude_resources: frozenset,
  include_namespaces: frozenset,
  exclude_namespaces: frozenset,
+ rules: list,
  dry_run: bool):
 
+ counter = Counter()
+
  for namespace in Namespace.objects(api):
  if matches_resource_filter(namespace, include_resources, exclude_resources, include_namespaces, exclude_namespaces):
- handle_resource(namespace, dry_run)
+ counter.update(handle_resource(namespace, rules, dry_run))
  else:
  logger.debug(f'Skipping {namespace.kind} {namespace}')
 
@@ -99,4 +118,7 @@ def clean_up(api,
  logger.error(f'Could not list {_type.kind} objects: {e}')
 
  for resource in filtered_resources:
- handle_resource(resource, dry_run)
+ counter.update(handle_resource(resource, rules, dry_run))
+
+ stats = ', '.join([f'{k}={v}' for k, v in counter.items()])
+ logger.info(f'Clean up run completed: {stats}')

kube_janitor/main.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 
 import time
-
 import logging
 
 from kube_janitor import __version__, cmd, shutdown
 from kube_janitor.helper import get_kube_api
 from kube_janitor.janitor import clean_up
+from kube_janitor.rules import load_rules_from_file
 
 logger = logging.getLogger('janitor')
 
@@ -23,12 +23,18 @@ def main():
  if args.dry_run:
  logger.info('**DRY-RUN**: no deletions will be performed!')
 
+ if args.rules_file:
+ rules = load_rules_from_file(args.rules_file)
+ logger.info(f'Loaded {len(rules)} rules from file {args.rules_file}')
+ else:
+ rules = []
+
  return run_loop(args.once, args.include_resources, args.exclude_resources, args.include_namespaces,
- args.exclude_namespaces, args.interval, args.dry_run)
+ args.exclude_namespaces, rules, args.interval, args.dry_run)
 
 
 def run_loop(run_once, include_resources, exclude_resources, include_namespaces, exclude_namespaces,
- interval, dry_run):
+ rules, interval, dry_run):
  handler = shutdown.GracefulShutdown()
  while True:
  try:
@@ -39,6 +45,7 @@ def run_loop(run_once, include_resources, exclude_resources, include_namespaces,
  exclude_resources=frozenset(exclude_resources.split(',')),
  include_namespaces=frozenset(include_namespaces.split(',')),
  exclude_namespaces=frozenset(exclude_namespaces.split(',')),
+ rules=rules,
  dry_run=dry_run)
  except Exception as e:
  logger.exception('Failed to clean up: %s', e)

kube_janitor/rules.py

@@ -0,0 +1,65 @@
+import collections
+import jmespath
+import logging
+import re
+import yaml
+
+from pykube.objects import NamespacedAPIObject
+
+from .helper import parse_ttl
+
+RULE_ID_PATTERN = re.compile(r'^[a-z][a-z0-9-]*$')
+
+logger = logging.getLogger(__name__)
+
+
+class Rule(collections.namedtuple('Rule', ['id', 'resources', 'jmespath', 'ttl'])):
+
+ def from_entry(entry: dict):
+ id_ = entry['id']
+ if not RULE_ID_PATTERN.match(id_):
+ raise ValueError(f'Invalid rule ID "{id_}": it has to match ^[a-z][a-z0-9-]*$')
+
+ # check whether TTL format is correct
+ parse_ttl(entry['ttl'])
+ return Rule(
+ id=id_,
+ resources=frozenset(entry['resources']),
+ jmespath=jmespath.compile(entry['jmespath']),
+ ttl=entry['ttl'])
+
+ def matches(self, resource: NamespacedAPIObject):
+ if resource.endpoint not in self.resources and '*' not in self.resources:
+ return False
+
+ result = self.jmespath.search(resource.obj)
+ logger.debug(f'Rule {self.id} with JMESPath "{self.jmespath.expression}" evaluated for {resource.kind} {resource.namespace}/{resource.name}: {result}')
+ return bool(result)
+
+
+def load_rules_from_file(filename: str):
+ with open(filename) as fd:
+ data = yaml.safe_load(fd)
+
+ try:
+ entries = data['rules']
+ except (TypeError, KeyError):
+ raise KeyError('The rules YAML file must have a top-level mapping with the key "rules"')
+
+ rules = []
+
+ for i, entry in enumerate(entries):
+ try:
+ if not isinstance(entry, dict):
+ raise TypeError('rule must be a mapping')
+
+ missing_keys = frozenset(Rule._fields) - entry.keys()
+ if missing_keys:
+ raise ValueError(f'rule is missing required keys: {missing_keys}')
+
+ rule = Rule.from_entry(entry)
+ rules.append(rule)
+ except Exception as e:
+ raise TypeError(f'Failed to load rule #{i} from file "{filename}": {e}')
+
+ return rules

tests/test_clean_up.py

@@ -33,7 +33,7 @@ def get(**kwargs):
  response.json.return_value = data
  return response
  api_mock.get = get
- clean_up(api_mock, ALL, [], ALL, ['kube-system'], dry_run=False)
+ clean_up(api_mock, ALL, [], ALL, ['kube-system'], [], dry_run=False)
 
 
 def test_clean_up_custom_resource():
@@ -61,7 +61,7 @@ def get(**kwargs):
  return response
 
  api_mock.get = get
- clean_up(api_mock, ALL, [], ALL, [], dry_run=False)
+ clean_up(api_mock, ALL, [], ALL, [], [], dry_run=False)
 
  # verify that the delete call happened
  api_mock.delete.assert_called_once_with(namespace='ns-1', url='customfoos/foo-1', version='srcco.de/v1')

tests/test_cmd.py

@@ -0,0 +1,6 @@
+from kube_janitor.cmd import get_parser
+
+
+def test_parse_args():
+ parser = get_parser()
+ parser.parse_args(['--dry-run', '--rules-file=/config/rules.yaml'])