cloud-config-elasticsearch.yml

#cloud-config

# Instance
ssh_authorized_keys:
  - %(LOCAL_SSH_KEY)s
bootcmd:
- set -ex
- cloud-init-per once ssh-users-ca echo "TrustedUserCAKeys /etc/ssh/users_ca.pub" >> /etc/ssh/sshd_config
# - cloud-init-per once accepted-oracle-license-v1-1 echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
- cloud-init-per once fallocate-swapfile fallocate -l 4G /swapfile
- cloud-init-per once chmod-swapfile chmod 600 /swapfile
- cloud-init-per once mkswap-swapfile mkswap /swapfile
package_upgrade: true

# install_packages_all
- set -ex
- sleep 30
- sudo apt-get update
- sudo apt-get -y install awscli
- sudo apt-get -y install git
- sudo apt-get -y install build-essential
- sudo apt-get -y install unattended-upgrades
- sudo apt-get -y install libbz2-dev
- sudo apt-get -y install libdb-dev
- sudo apt-get -y install libffi-dev
- sudo apt-get -y install libgdbm-dev
- sudo apt-get -y install liblzma-dev
- sudo apt-get -y install libncurses5-dev
- sudo apt-get -y install libnss3-dev
- sudo apt-get -y install libreadline-dev
- sudo apt-get -y install libssl-dev
- sudo apt-get -y install libsqlite3-dev
- sudo apt-get -y install python2
- sudo apt-get -y install python3-dev
- sudo apt-get -y install python3-pip
- sudo apt-get -y install python3-venv
- sudo apt-get -y install python3-testresources
- sudo apt-get -y install zlib1g-dev
- sudo apt-get -y install redis-server
- sudo apt-get -y install nagios-nrpe-plugin
- sudo apt-get -y install monitoring-plugins
- sudo apt-get -y install nagios-nrpe-server
- sudo apt-get -y install libjson-perl
- sudo apt-get -y install libmonitoring-plugin-perl
- sudo apt-get -y install liblwp-useragent-determined-perl
- sudo apt-get -y install sysstat
- sudo apt-get -y install libarchive-tools

output:
  all: '| tee -a /var/log/cloud-init-output.log'

# Manually install java
- sudo -u ubuntu aws s3 cp --region=us-west-2 --recursive s3://t2depi-conf-prod/.aws ~ubuntu/.aws
- sudo mkdir -p /usr/lib/jvm
- sudo chmod 777 /usr/lib/jvm
- sudo tar -xzvf /home/ubuntu/.aws/jdk-11.0.3_linux-x64_bin.tar.gz --directory /usr/lib/jvm/
- sudo -u ubuntu rm -r /home/ubuntu/.aws
- sudo chmod 755 /usr/lib/jvm
- sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk-11.0.3/bin/java 100
- sudo -u ubuntu mv /home/ubuntu/.ssh/authorized_keys /home/ubuntu/.ssh/authorized_keys2
- sudo -u ubuntu aws s3 cp --region=us-west-2 s3://t2depi-conf-prod/ssh-keys/prod-authorized_keys/authorized_keys /home/ubuntu/.ssh/authorized_keys
# install_and_setup_elasticsearch
- sudo wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo tee /etc/apt/es_gpg_key
- sudo apt-key add /etc/apt/es_gpg_key
- echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list
- sudo apt-get -y install apt-transport-https
- sudo apt update
- yes |sudo apt-get -y install elasticsearch
# Ideally this would build as a different user so encoded only has read
# permissions
- MEMGIGS=$(awk '/MemTotal/{printf int($2 / 1024**2)}' /proc/meminfo)
- if [ "$MEMGIGS" -gt 12 ];
- then
-    echo "ES_HEAP_SIZE=8g" > /etc/default/elasticsearch
- else
-    echo "ES_HEAP_SIZE=4g" > /etc/default/elasticsearch
-    sysctl "vm.swappiness=1"
-    swapon /swapfile
- fi
- update-rc.d elasticsearch defaults
- sudo bash /etc/elasticsearch/cluster.sh %(CLUSTER_NAME)s
- sudo /usr/share/elasticsearch/bin/plugin install discovery-ec2
- service elasticsearch start


users:
- default
- name: build
  gecos: Build user
  inactive: true
  system: true
- name: snowflakes
  gecos: Snowflake Database daemon user
  inactive: true
  system: true
  # Specified homedir must exist
  # https://github.com/rubygems/rubygems/issues/689
  homedir: /srv/snowflakes

write_files:
- path: /etc/apt/apt.conf.d/20auto-upgrades
  content: |
    APT::Periodic::Update-Package-Lists "1";
    APT::Periodic::Unattended-Upgrade "1";

- path: /etc/apt/apt.conf.d/50unattended-upgrades
  content: |
    Unattended-Upgrade::Allowed-Origins {
        "${distro_id} ${distro_codename}-security";
    };
    Unattended-Upgrade::Automatic-Reboot "true";

- path: /etc/cron.d/cloudwatchmon
  content: |
    */5 * * * * nobody /opt/cloudwatchmon/bin/mon-put-instance-stats.py --mem-util --swap-util --disk-space-util --disk-path=/ --from-cron

- path: /etc/elasticsearch/elasticsearch.yml
  content: |
    index.search.slowlog.threshold.query.warn: 8s
    index.search.slowlog.threshold.fetch.warn: 1s
    index.indexing.slowlog.threshold.index.warn: 25s
    network.host: 0.0.0.0
    action.auto_create_index: false
    index.mapper.dynamic: false

    node.master: false
    node.data: true

    discovery:
        type: ec2

    cloud.aws.region: us-west-2

    discovery.ec2.groups: elasticsearch-https, ssh-http-https

    indices.query.bool.max_clause_count: 8192

    # discovery.zen.minimum_master_nodes: 2

- path: /etc/elasticsearch/cluster.sh
  content: |
    #!/bin/bash
    name=$1

    if [[ -n "$name" ]]; then
        echo "cluster.name: $name" >> /etc/elasticsearch/elasticsearch.yml
    else
        echo "argument error"
    fi

- path: /etc/ssh/users_ca.pub
  content: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv/ymOcnN4LhM4NACc3Or116XXJ6KytuOgB/+1qNkOFBqBosrn7cmJ35rsoNHRgYNrCsRE9ch74RKsN6H72FtSJgBhGh/9oUK7Os6Fqt3/ZZXxgxIx6ubs/MTgrxrAnujiBxUXMXQhLKMriNMpo8mt4nGYVtLk9PBjiyfncaS8H9ZKoNio9dhP8bmTuYvioAI35dqKdSlVLyzr/XkZxia8Ki+pQ0N6uuiEwMR3ToM+LSp8wpFOOAiu4PEAujRW7us/+1hlpKWfn0J7/V3826joHE+I967Vg/+ikcVhF77JjK1nib879VgCWfmn1HPQosIpk4yJfVgGvRVI7I2nfBPVw== snowflakes@snowflakes.org