Merge pull request #1324 from hacktobeer/docker-optimize

Optimize Turbinia worker and server docker image.
google · Aug 9, 2023 · 9713212 · 9713212
2 parents 136f93a + 1d5a24a
commit 9713212
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 55 deletions.
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
@@ -1,8 +1,8 @@
 FROM ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive 
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get -y install \
+ENV PIP_NO_CACHE_DIR=1
+RUN apt-get update && apt-get -y upgrade && apt-get -y install \
     apt-transport-https \
     apt-utils \
     ca-certificates \
@@ -11,19 +11,17 @@ RUN apt-get -y install \
     python3-pip \
     software-properties-common \
     sudo \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*
 
-RUN pip3 install pip --upgrade
-RUN pip3 install urllib3 cryptography requests --upgrade
+RUN pip3 install pip --upgrade \
+    && pip3 install urllib3 cryptography requests --upgrade
 
-ADD requirements.txt /tmp/
+ADD . /tmp/
 RUN cd /tmp/ && pip3 install -r requirements.txt
 
-ADD . /tmp/
 # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
-RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi
-
-RUN cd /tmp/ && python3 setup.py install
+RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \
+    && cd /tmp/ && python3 setup.py install
 
 RUN useradd -r -s /bin/nologin -u 999 turbinia
 

diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile
@@ -1,42 +1,80 @@
-FROM ubuntu:22.04
+# Multi-stage build
+# Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable] GOVERSION=[1.20.6|1.18|?]"
+
+# Build 0 - fraken
+FROM golang:alpine AS fraken-builder
+RUN apk add --no-cache -t .build-deps \
+  autoconf \
+  automake \
+  bison \
+  build-base \
+  curl \
+  file \
+  file-dev \
+  flex \
+  git \
+  jansson \
+  jansson-dev \
+  jansson-static \
+  libc-dev \
+  libmagic \
+  libmagic-static \
+  libtool \
+  linux-headers \
+  openssl \
+  openssl-dev \
+  openssl-libs-static \
+  py3-setuptools \
+  python3 \
+  python3-dev \
+  sudo
+
+RUN set -x \
+  && echo "Compiling Yara from source..." 
+
+# Fetch and compile libyara
+RUN mkdir -p /opt/fraken/yara
+WORKDIR /opt/fraken/yara 
+RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')"  | tar -xz --strip-components=1
+RUN ./bootstrap.sh 
+RUN sync 
+RUN ./configure --with-crypto \
+  --enable-magic \
+  --enable-cuckoo 
+RUN make 
+RUN sudo make install 
+
+# Compile fraken statically
+COPY tools/fraken/* /opt/fraken/
+RUN cd /opt/fraken && GOOS=linux GOARCH=amd64 go build -a -v -ldflags="-linkmode=external -extldflags=-static" -installsuffix netgo -tags yara_static,osusergo,netgo -o fraken
+
+# Build 1 - Turbinia Worker
+FROM ubuntu:22.04 AS worker-builder
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_NO_CACHE_DIR=1
 
-# Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable]"
 ARG PPA_TRACK=stable
 
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get -y install \
+RUN apt-get update && apt-get -y upgrade && apt-get -y install \
     apt-transport-https \
     apt-utils \
-    automake \
     ca-certificates \
     curl \
-    dh-autoreconf \
-    gcc \
     git \
-    golang \
-    gpg \
-    go-bindata \
     john \
     john-data \
     hashcat \
     hashcat-data \
-    libjemalloc-dev \
     libleveldb1d \
     libleveldb-dev \
-    libprotobuf-c-dev \
-    libssl-dev \
     libterm-readline-gnu-perl \
-    libtool \
     lvm2 \
-    make \
-    pkg-config \
     python3-pip \
     software-properties-common \
     sudo \
     testdisk \
     wget \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*
 
 ADD requirements.txt /tmp/
 RUN cd /tmp/ && pip3 install -r requirements.txt
@@ -45,10 +83,13 @@ RUN pip3 install pip --upgrade
 RUN pip3 install requests --upgrade
 RUN pip3 install urllib3 cryptography --upgrade
 
-# Install third-party worker dependencies
+# Install third-party dependencies
+#   dfwdewey
+#   pyhindsight
+#   impacket
 RUN pip3 install dfDewey
-# TODO(hacktobeer) uncomment when protobuf lib dependency if fixed upstream
-# RUN pip3 install pyhindsight
+RUN pip3 install pyhindsight
+RUN pip3 install impacket --no-deps
 
 # Install various packages from the GIFT PPA
 #   bulkextractor
@@ -59,7 +100,6 @@ RUN pip3 install dfDewey
 #   libluksde-tools
 #   Plaso
 #   Sleuthkit
-
 RUN add-apt-repository -y ppa:gift/$PPA_TRACK
 RUN apt-get update && apt-get -y install \
     bulk-extractor \
@@ -76,53 +116,50 @@ RUN apt-get update && apt-get -y install \
     python3-dfvfs \
     python3-plaso \
     sleuthkit \
-    --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60
+    && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*
 
+# Add turbinia user to system and sudoers
 RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia
 RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia
 
-RUN pip3 install impacket --no-deps
-
+# Install yara rules and fraken binary.
 RUN cd /opt \
     && git clone https://github.com/Neo23x0/signature-base.git \
-    && sudo chown -R turbinia:turbinia /opt/signature-base
-
+    && sudo chown -R turbinia:turbinia /opt/signature-base \
+    && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
 COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
+# Install fraken binary from multi-stage build 
+RUN mkdir -p /opt/fraken
+COPY --chown=turbinia:turbinia --from=fraken-builder /opt/fraken/fraken /opt/fraken/fraken
 
-RUN mkdir -p /opt/fraken/yara && chown -R turbinia:turbinia /opt/fraken
-COPY --chown=turbinia:turbinia tools/fraken/* /opt/fraken/
-RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')"  | tar -xz --strip-components=1
-RUN cd /opt/fraken/yara && ./bootstrap.sh \
-    && ./configure \
-    && make && sudo make install && sudo ldconfig
-RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken
+# Install container-explorer
+RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh
+RUN chmod +x /tmp/container-explorer-setup.sh
+RUN sudo /tmp/container-explorer-setup.sh install
 
+# Setup turbinia user folders and permissions
 RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \
     && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \
     && mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \
     && chown -R turbinia:turbinia /var/lib/turbinia/ \
     && chown -R turbinia:turbinia /var/log/turbinia/ \
     && mkdir -p /home/turbinia && chown -R turbinia:turbinia /home/turbinia
 
-# Get a decent password list
+# Get a decent password list for john/hashcat
 RUN cd /home/turbinia && echo "" > password.lst
 RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/285474cf9bff85f3323c5a1ae436f78acd1cb62c/Passwords/UserPassCombo-Jay.txt >> password.lst
 RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/Common-Credentials/10-million-password-list-top-1000000.txt >> password.lst
 RUN cp /home/turbinia/password.lst /root/
 
 # Copy Kubernetes support tool to home folder
-COPY k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py
-RUN chown turbinia:turbinia /home/turbinia/check-lockfile.py
-
-# Install container-explorer
-RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh
-RUN chmod +x /tmp/container-explorer-setup.sh
-RUN sudo /tmp/container-explorer-setup.sh install
+COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py
 
-ADD . /tmp/
+# Install Turbinia
 # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
-RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi
-RUN cd /tmp/ && python3 setup.py install
+# install turbinia and cleanup /tmp
+ADD . /tmp/
+RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \
+    && cd /tmp/ && python3 setup.py install
 
 COPY docker/worker/start.sh /home/turbinia/start.sh
 RUN chmod +rwx /home/turbinia/start.sh