Docker overhaul

outflanknl · Oct 22, 2020 · 9c8df71 · 9c8df71
1 parent 965ffc9
commit 9c8df71
Show file tree

Hide file tree

Showing 95 changed files with 45,913 additions and 122 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-RedELK version 2 - BETA 2
+RedELK version 2 - BETA 3 (unreleased)
diff --git a/c2servers/install-c2server-cobaltstrike.sh b/c2servers/install-c2server-cobaltstrike.sh
@@ -41,6 +41,7 @@ preinstallcheck() {
     fi
 }
 
+printf "`date +'%b %e %R'` $INSTALLER - Starting installer\n" > $LOGFILE 2>&1
 echo ""
 echo ""
 echo ""
@@ -53,8 +54,13 @@ echo ""
 echo ""
 echo ""   
 echo "This script will install and configure necessary components for RedELK on Cobalt Strike teamservers"
-echo "`date +'%b %e %R'` $INSTALLER - Starting installer" | tee $LOGFILE
-printf "`date +'%b %e %R'` $INSTALLER - Starting installer\n" > $LOGFILE 2>&1
+echo ""
+echo ""
+
+if [[ $EUID -ne 0 ]]; then
+  echo "[X] Not running as root. Exiting"
+  exit 1
+fi
 
 if ! [ $# -eq 3 ] ; then
     echo "[X] ERROR Incorrect amount of parameters"

diff --git a/c2servers/install-c2server-poshc2.sh b/c2servers/install-c2server-poshc2.sh
@@ -42,6 +42,7 @@ preinstallcheck() {
     fi
 }
 
+printf "`date +'%b %e %R'` $INSTALLER - Starting installer\n" > $LOGFILE 2>&1
 echo ""
 echo ""
 echo ""
@@ -54,8 +55,13 @@ echo ""
 echo ""
 echo ""   
 echo "This script will install and configure necessary components for RedELK on PoshC2 teamservers"
-echo "`date +'%b %e %R'` $INSTALLER - Starting installer" | tee $LOGFILE
-printf "`date +'%b %e %R'` $INSTALLER - Starting installer\n" > $LOGFILE 2>&1
+echo ""
+echo ""
+
+if [[ $EUID -ne 0 ]]; then
+  echo "[X] Not running as root. Exiting"
+  exit 1
+fi
 
 if ! [ $# -eq 3 ] ; then
     echo "[X] ERROR Incorrect amount of parameters"

diff --git a/elkserver/docker/docker-compose-limited.yml b/elkserver/docker/docker-compose-limited.yml
@@ -19,11 +19,15 @@ volumes:
     driver: local
   redelk_kibana_data:
     driver: local
+  redelk_jupyter_data:
+    driver: local
+  redelk_bloodhound_data:
+    driver: local
 
 services:
   elasticsearch:
     container_name: redelk-elasticsearch
-    image: marcoverip/redelk_elasticsearch:V2beta2
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
     networks:
       - dockernetredelk
     expose:
@@ -35,8 +39,8 @@ services:
       - cluster.name=redelk-cluster
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
-    #  - xpack.security.enabled=false
-    #  - xpack.watcher.enabled=false
+      - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
     restart: always
 
   logstash:
@@ -51,7 +55,7 @@ services:
       - ./redelk-logstash/live/config/conf.d:/etc/logstash/conf.d
       - ./redelk-logstash/live/config/certs:/etc/logstash/certs
       - ./redelk-logstash/live/config/ruby-scripts:/etc/logstash/ruby-scripts
-      - ./redelk-base/live/redelklogs/redteamdomaincheck.txt:/var/log/redelk/redteamdomaincheck.txt
+      - ./redelk-base/live/redelklogs:/var/log/redelk
       - redelk_logstash_data:/usr/share/logstash/data
     environment:
       - node.name=redelk-logstash
@@ -79,32 +83,32 @@ services:
     depends_on:
       - logstash
 
-  base:
-    container_name: redelk-base
-    image: marcoverip/redelk_base:V2beta2
+  nginx:
+    container_name: redelk-nginx
+    image: nginx:latest
     networks:
       - dockernetredelk
     volumes:
+      - ./redelk-nginx/live/config:/etc/nginx/conf.d
       - ./redelk-base/live/var_www_html:/var/www/html
-      - ./redelk-base/live/config/cron.d/redelk:/etc/cron.d/redelk
-      - ./redelk-base/live/config/etc/redelk:/etc/redelk
-      - ./redelk-base/live/ssh:/home/scponly/.ssh
-      - ./redelk-base/live/redelklogs:/var/log/redelk
+    ports:
+      - "80:80"
+      - "443:443"
+    restart: always
     depends_on:
       - kibana
 
-  nginx:
-    container_name: redelk-nginx
-    image: nginx:latest
+  base:
+    container_name: redelk-base
+    build: ./redelk-base
     networks:
       - dockernetredelk
     volumes:
-      - ./redelk-nginx/config/default.conf:/etc/nginx/conf.d/default.conf
-      - ./redelk-nginx/config/htpasswd.users:/etc/nginx/htpasswd.users
       - ./redelk-base/live/var_www_html:/var/www/html
-    ports:
-      - "80:80"
-      - "443:443"
+      - ./redelk-base/live/config/etc/cron.d:/etc/cron.d
+      - ./redelk-base/live/config/etc/redelk:/etc/redelk
+      - ./redelk-base/live/redelklogs:/var/log/redelk
+      - ./redelk-base/live/ssh:/home/redelk/.ssh
     restart: always
     depends_on:
-      - base
+      - nginx
diff --git a/elkserver/docker/docker-compose.yml b/elkserver/docker/docker-compose.yml
@@ -24,11 +24,10 @@ volumes:
   redelk_bloodhound_data:
     driver: local
 
-
 services:
   elasticsearch:
     container_name: redelk-elasticsearch
-    image: marcoverip/redelk_elasticsearch:V2beta2
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
     networks:
       - dockernetredelk
     expose:
@@ -40,8 +39,8 @@ services:
       - cluster.name=redelk-cluster
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
-    #  - xpack.security.enabled=false
-    #  - xpack.watcher.enabled=false
+      - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
     restart: always
 
   logstash:
@@ -56,7 +55,7 @@ services:
       - ./redelk-logstash/live/config/conf.d:/etc/logstash/conf.d
       - ./redelk-logstash/live/config/certs:/etc/logstash/certs
       - ./redelk-logstash/live/config/ruby-scripts:/etc/logstash/ruby-scripts
-      - ./redelk-base/live/redelklogs/redteamdomaincheck.txt:/var/log/redelk/redteamdomaincheck.txt
+      - ./redelk-base/live/redelklogs:/var/log/redelk
       - redelk_logstash_data:/usr/share/logstash/data
     environment:
       - node.name=redelk-logstash
@@ -84,69 +83,65 @@ services:
     depends_on:
       - logstash
 
-  jupyter:
-    container_name: redelk-jupyter
-    image: jupyter/scipy-notebook:4a112c0f11eb
+  nginx:
+    container_name: redelk-nginx
+    image: nginx:latest
     networks:
       - dockernetredelk
-    expose:
-      - "8888"
     volumes:
-      - ./redelk-jupyter/live/workbooks:/home/jovyan/work
-    environment:
-      - NotebookApp.token=''
-      - NotebookApp.password=''
-      - NotebookApp.allow_remote_access='True'
-      - NotebookApp.allow_origin='*'
-      - NotebookApp.base_url='/jupyter/'
+      - ./redelk-nginx/live/config:/etc/nginx/conf.d
+      - ./redelk-base/live/var_www_html:/var/www/html
+    ports:
+      - "80:80"
+      - "443:443"
     restart: always
     depends_on:
       - kibana
 
   base:
     container_name: redelk-base
-    image: marcoverip/redelk_base:V2beta2
+    build: ./redelk-base
+    # image: outflanknl/redelk_base:V2beta2
     networks:
       - dockernetredelk
     volumes:
       - ./redelk-base/live/var_www_html:/var/www/html
-      - ./redelk-base/live/config/cron.d/redelk:/etc/cron.d/redelk
+      - ./redelk-base/live/config/etc/cron.d:/etc/cron.d
       - ./redelk-base/live/config/etc/redelk:/etc/redelk
-      - ./redelk-base/live/ssh:/home/scponly/.ssh
       - ./redelk-base/live/redelklogs:/var/log/redelk
+      - ./redelk-base/live/ssh:/home/redelk/.ssh
+    restart: always
+    depends_on:
+      - nginx
+
+  jupyter:
+    container_name: redelk-jupyter
+    build: ./redelk-jupyter
+    networks:
+      - dockernetredelk
+    expose:
+      - "8888"
+    volumes:
+      - ./redelk-jupyter/live/workbooks:/home/jovyan/work
+    restart: always
     depends_on:
-      - jupyter
+      - base
 
   bloodhound:
-    container_name: redelk-bloodbound
+    container_name: redelk-bloodhound
     image: specterops/bloodhound-neo4j
     networks:
       - dockernetredelk
     volumes:
       - redelk_bloodhound_data:/var/lib/neo4j
     ports:
-      - 7687:7687
-      - 7474:7474
+      - "7687:7687"
+    expose: 
+      - "7474"
     environment:
       - NEO4J_AUTH=neo4j/BloodHound
       - NEO4J_dbms_memory_heap_initial__size=1G
       - NEO4J_dbms_memory_heap_max__size=1G
       - NEO4J_dbms_memory_pagecache_size=1G
     depends_on:
-      - base
-
-  nginx:
-    container_name: redelk-nginx
-    image: nginx:latest
-    networks:
-      - dockernetredelk
-    volumes:
-      - ./redelk-nginx/live/config/default.conf:/etc/nginx/conf.d/default.conf
-      - ./redelk-nginx/live/config/htpasswd.users:/etc/nginx/htpasswd.users
-      - ./redelk-base/live/var_www_html:/var/www/html
-    ports:
-      - "80:80"
-      - "443:443"
-    restart: always
-    depends_on:
-      - base
+      - jupyter
diff --git a/elkserver/docker/redelk-base/Dockerfile b/elkserver/docker/redelk-base/Dockerfile
@@ -0,0 +1,36 @@
+#
+# Part of RedELK
+# Dockerfile for RedELK base image
+#
+# Author: Outflank B.V. / Marc Smeets
+#
+
+FROM phusion/baseimage:18.04-1.0.0
+LABEL maintainer="Outflank B.V. / Marc Smeets"
+LABEL description="RedELK Base Image"
+
+# Copy relevant install data
+RUN mkdir -p /root/redelkinstalldata/
+COPY ./redelkinstalldata/ /root/redelkinstalldata/
+
+# Init script
+RUN mkdir -p /etc/my_init.d
+RUN cp /root/redelkinstalldata/42_redelk-base-docker-init.sh /etc/my_init.d/42_redelk-base-docker-init.sh
+RUN chmod +x /etc/my_init.d/42_redelk-base-docker-init.sh
+
+# copy relevant scripts to redelk script working dir
+RUN mkdir -p /usr/share/redelk/bin
+RUN cp -r /root/redelkinstalldata/scripts/* /usr/share/redelk/bin/
+RUN chmod -R 775 /usr/share/redelk/bin/*
+
+# Install required packages and python requirements
+RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update > /dev/null
+RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y install rsync python3-pil python3-pip > /dev/null
+RUN mkdir -p /usr/share/redelk/bin
+RUN cp -r /root/redelkinstalldata/scripts/ /usr/share/redelk/bin/
+RUN pip3 install -q -r /usr/share/redelk/bin/Chameleon/requirements.txt > /dev/null
+RUN pip3 install -q elasticsearch pymsteams > /dev/null
+
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+CMD ["/sbin/my_init"]
diff --git a/elkserver/docker/redelk-base/live/config/etc/cron.d/redelk b/elkserver/docker/redelk-base/live/config/etc/cron.d/redelk
@@ -0,0 +1,41 @@
+# Part of RedELK
+# cron.d script for periodic actions related to RedELK
+#
+# Author: Outflank B.V. / Marc Smeets
+#
+
+SHELL=/bin/sh
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+# RedELK rsync comands for getting remote log files from teamservers via rsync - replace $IP with the remote host's DNS/IP, $HOSTNAME with the remote hostname as configured in filebeat on that system amd $USER with the username to login with.
+# Repeat line for multiple teamservers.
+# m h dom mon dow user  command
+#*/2 * * * * redelk /usr/share/redelk/bin/getremotelogs.sh $IP $HOSTNAME scponly
+#
+
+# Run update script for TOR exit nodes twice a day
+# m h dom mon dow user  command
+00 00,12 * * * redelk /usr/share/redelk/bin/run_torexitnodeupdate.sh
+
+# Run update script for SSLBL Botnet C2 IP addresses every 30min
+# m h dom mon dow user  command
+*/30 * * * * redelk /usr/share/redelk/bin/run_abusechbotnetc2ipupdate.sh
+
+# Run update script for SSLBL SSL Certificate Blacklis
+# m h dom mon dow user  command
+*/30 * * * * redelk /usr/share/redelk/bin/run_abusechsslcertupdate.sh
+
+# Run update script for rogue domain list
+# And run Chameleon.py for remote checking of domain classification
+# m h dom mon dow user  command
+*/30 * * * * redelk /usr/share/redelk/bin/run_roguedomainsupdate.sh
+
+# Start script to create thumbnails of received screenshots
+# m h dom mon dow user  command
+*/2 * * * * redelk /usr/share/redelk/bin/makethumbnail.py /var/www/html/c2logs/
+
+# Start the enrichment of beacon data in ELK
+* * * * * redelk /usr/share/redelk/bin/run_enrich.sh 
+
+# Start the checks for alarms
+*/5 * * * * redelk /usr/share/redelk/bin/run_alarm.sh