marbot-elasticsearch.yml

---
# Copyright widdix GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
AWSTemplateFormatVersion: '2010-09-09'
Description: 'marbot.io: Elasticsearch/OpenSearch domain monitoring (https://github.com/marbot-io/monitoring-jump-start)'
Metadata:
  'AWS::CloudFormation::Interface':
    ParameterGroups:
    - Label:
        default: 'marbot endpoint'
      Parameters:
      - EndpointId
      - Stage
    - Label:
        default: 'Elasticsearch'
      Parameters:
      - DomainName
    - Label:
        default: 'Thresholds'
      Parameters:
      - CPUUtilizationThreshold
      - CPUCreditBalanceThreshold
      - MasterCPUUtilizationThreshold
      - MasterCPUCreditBalanceThreshold
      - FreeStorageSpaceThreshold
      - JVMMemoryPressureThreshold
      - MasterJVMMemoryPressureThreshold
Parameters:
  EndpointId:
    Description: 'Your marbot endpoint ID (to get this value: select a channel where marbot belongs to and send a message like this: "@marbot show me my endpoint id").'
    Type: String
  Stage:
    Description: 'marbot stage (never change this!).'
    Type: String
    Default: v1
    AllowedValues: [v1, dev]
  DomainName:
    Description: 'The domain name of the Elasticsearch domain that you want to monitor.'
    Type: String
  CPUUtilizationThreshold:
    Description: 'The maximum percentage of CPU resources used for data nodes in the cluster (set to -1 to disable).'
    Type: Number
    Default: 80
    MinValue: -1
    MaxValue: 100
  CPUCreditBalanceThreshold:
    Description: 'The minimum number of CPU credits used for data nodes in the cluster (t* instances only; set to -1 to disable).'
    Type: Number
    Default: 20
    MinValue: -1
  MasterCPUUtilizationThreshold:
    Description: 'The maximum percentage of CPU resources used for master nodes in the cluster (set to -1 to disable).'
    Type: Number
    Default: 50
    MinValue: -1
    MaxValue: 100
  MasterCPUCreditBalanceThreshold:
    Description: 'The minimum number of CPU credits used for master nodes in the cluster (t* instances only; set to -1 to disable).'
    Type: Number
    Default: 20
    MinValue: -1
  FreeStorageSpaceThreshold:
    Description: 'The minimum amount of available storage space in the cluster in Megabyte (set to -1 to disable).'
    Type: Number
    Default: 2000 # 2 Gigabyte in Megabyte
    MinValue: -1
  JVMMemoryPressureThreshold:
    Description: 'The maximum percentage of the Java heap used for all data nodes in the cluster (set to -1 to disable).'
    Type: Number
    Default: 80
    MinValue: -1
    MaxValue: 100
  MasterJVMMemoryPressureThreshold:
    Description: 'The maximum percentage of the Java heap used for all master nodes in the cluster (set to -1 to disable).'
    Type: Number
    Default: 80
    MinValue: -1
    MaxValue: 100
Conditions:
  HasCPUUtilizationThreshold: !Not [!Equals [!Ref CPUUtilizationThreshold, '-1']]
  HasCPUCreditBalanceThreshold: !Not [!Equals [!Ref CPUCreditBalanceThreshold, '-1']]
  HasMasterCPUUtilizationThreshold: !Not [!Equals [!Ref MasterCPUUtilizationThreshold, '-1']]
  HasMasterCPUCreditBalanceThreshold: !Not [!Equals [!Ref MasterCPUCreditBalanceThreshold, '-1']]
  HasFreeStorageSpaceThreshold: !Not [!Equals [!Ref FreeStorageSpaceThreshold, '-1']]
  HasJVMMemoryPressureThreshold: !Not [!Equals [!Ref JVMMemoryPressureThreshold, '-1']]
  HasMasterJVMMemoryPressureThreshold: !Not [!Equals [!Ref MasterJVMMemoryPressureThreshold, '-1']]
Resources:
  ##########################################################################
  #                                                                        #
  #                                 TOPIC                                  #
  #                                                                        #
  ##########################################################################
  Topic:
    Type: 'AWS::SNS::Topic'
    Properties: {}
  TopicPolicy:
    Type: 'AWS::SNS::TopicPolicy'
    Properties:
      PolicyDocument:
        Id: Id1
        Version: '2012-10-17'
        Statement:
        - Sid: Sid1
          Effect: Allow
          Principal:
            Service: 'events.amazonaws.com' # Allow EventBridge
          Action: 'sns:Publish'
          Resource: !Ref Topic
        - Sid: Sid2
          Effect: Allow
          Principal:
            AWS: '*' # Allow CloudWatch Alarms
          Action: 'sns:Publish'
          Resource: !Ref Topic
          Condition:
            StringEquals:
              'AWS:SourceOwner': !Ref 'AWS::AccountId'
      Topics:
      - !Ref Topic
  TopicEndpointSubscription:
    DependsOn: TopicPolicy
    Type: 'AWS::SNS::Subscription'
    Properties:
      DeliveryPolicy:
        healthyRetryPolicy:
          minDelayTarget: 1
          maxDelayTarget: 60
          numRetries: 100
          numNoDelayRetries: 0
          backoffFunction: exponential
        throttlePolicy:
          maxReceivesPerSecond: 1
      Endpoint: !Sub 'https://api.marbot.io/${Stage}/endpoint/${EndpointId}'
      Protocol: https
      TopicArn: !Ref Topic
  MonitoringJumpStartEvent:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::Events::Rule'
    Properties:
      Description: 'Monitoring Jump Start connection. (created by marbot)'
      ScheduleExpression: 'rate(30 days)'
      State: ENABLED
      Targets:
      - Arn: !Ref Topic
        Id: marbot
        Input: !Sub |
          {
            "Type": "monitoring-jump-start-connection",
            "StackTemplate": "marbot-elasticsearch",
            "StackVersion": "1.5.0",
            "Partition": "${AWS::Partition}",
            "AccountId": "${AWS::AccountId}",
            "Region": "${AWS::Region}",
            "StackId": "${AWS::StackId}",
            "StackName": "${AWS::StackName}"
          }
  ##########################################################################
  #                                                                        #
  #                                 ALARMS                                 #
  #                                                                        #
  ##########################################################################
  StatusYellowAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Replica shards for at least one index are not allocated to nodes in a cluster. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'ClusterStatus.yellow'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Maximum
      Threshold: 0
  StatusRedAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Primary and replica shards of at least one index are not allocated to nodes in a cluster. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'ClusterStatus.red'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Maximum
      Threshold: 0
  CPUUtilizationTooHighAlarm:
    Condition: HasCPUUtilizationThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Average CPU utilization over last 10 minutes too high. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'CPUUtilization'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 600
      Statistic: Average
      Threshold: !Ref CPUUtilizationThreshold
  CPUCreditBalanceTooLowAlarm:
    Condition: HasCPUCreditBalanceThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Average CPU credit balance over last 10 minutes too low, expect a significant performance drop soon. (created by marbot)'
      ComparisonOperator: LessThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: CPUCreditBalance
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 600
      Statistic: Average
      Threshold: !Ref CPUCreditBalanceThreshold
  MasterCPUUtilizationTooHighAlarm:
    Condition: HasMasterCPUUtilizationThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Average CPU utilization over last 10 minutes too high. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'MasterCPUUtilization'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 600
      Statistic: Average
      Threshold: !Ref MasterCPUUtilizationThreshold
  MasterCPUCreditBalanceTooLowAlarm:
    Condition: HasMasterCPUCreditBalanceThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Average CPU credit balance over last 10 minutes too low, expect a significant performance drop soon. (created by marbot)'
      ComparisonOperator: LessThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: MasterCPUCreditBalance
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 600
      Statistic: Average
      Threshold: !Ref MasterCPUCreditBalanceThreshold
  FreeStorageSpaceTooLowAlarm:
    Condition: HasFreeStorageSpaceThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Cluster is running out of storage space. (created by marbot)'
      ComparisonOperator: LessThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'FreeStorageSpace'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Minimum
      Threshold: !Ref FreeStorageSpaceThreshold
  IndexWritesBlockedTooHighAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Cluster is blocking incoming write requests. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'ClusterIndexWritesBlocked'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Maximum
      Threshold: 0
  JVMMemoryPressureTooHighAlarm:
    Condition: HasJVMMemoryPressureThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Average JVM memory pressure over last 10 minutes too high. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'JVMMemoryPressure'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 600
      Statistic: Average
      Threshold: !Ref JVMMemoryPressureThreshold
  MasterJVMMemoryPressureTooHighAlarm:
    Condition: HasMasterJVMMemoryPressureThreshold
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Average JVM memory pressure over last 10 minutes too high. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'MasterJVMMemoryPressure'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 600
      Statistic: Average
      Threshold: !Ref MasterJVMMemoryPressureThreshold
  MasterNotReachableFromNodeAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Master node stopped or not reachable. Usually the result of a network connectivity issue or AWS dependency problem. (created by marbot)'
      ComparisonOperator: LessThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'MasterReachableFromNode'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Minimum
      Threshold: 1
  AutomatedSnapshotFailureTooHighAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'No automated snapshot was taken for the domain in the previous 36 hours. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'AutomatedSnapshotFailure'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Maximum
      Threshold: 0
  KibanaHealthyNodesTooLowAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'Kibana is inaccessible. (created by marbot)'
      ComparisonOperator: LessThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'KibanaHealthyNodes'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Minimum
      Threshold: 1
  KMSKeyErrorAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'KMS customer master key used to encrypt data at rest has been disabled. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'KMSKeyError'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Maximum
      Threshold: 0
  KMSKeyInaccessibleAlarm:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::CloudWatch::Alarm'
    Properties:
      AlarmActions:
      - !Ref Topic
      AlarmDescription: 'KMS customer master key used to encrypt data at rest has been deleted or revoked its grants to Amazon ES. (created by marbot)'
      ComparisonOperator: GreaterThanThreshold
      Dimensions:
      - Name: ClientId
        Value: !Ref 'AWS::AccountId'
      - Name: DomainName
        Value: !Ref DomainName
      EvaluationPeriods: 1
      MetricName: 'KMSKeyInaccessible'
      Namespace: 'AWS/ES'
      OKActions:
      - !Ref Topic
      Period: 60
      Statistic: Maximum
      Threshold: 0
  ##########################################################################
  #                                                                        #
  #                                 EVENTS                                 #
  #                                                                        #
  ##########################################################################
  ESSoftwareUpdateEvent:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::Events::Rule'
    Properties:
      Description: 'Alerts and notifications from Elasticsearch/OpenSearch software updates. (created by marbot)'
      EventPattern:
        source:
        - 'aws.es'
        'detail-type':
        - 'Amazon ES Service Software Update Notification'
        - 'Amazon OpenSearch Service Software Update Notification'
        resources:
        - !Sub 'arn:${AWS::Partition}:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}'
        detail:
          status:
          - Failed
          - Available
          - Completed
          - Required
      State: ENABLED
      Targets:
      - Arn: !Ref Topic
        Id: marbot
Outputs:
  StackName:
    Description: 'Stack name.'
    Value: !Sub '${AWS::StackName}'
  StackTemplate:
    Description: 'Stack template.'
    Value: 'marbot-elasticsearch'
  StackVersion:
    Description: 'Stack version.'
    Value: '1.5.0'