From d4cd5e2c6f16aa23ac32ea38f166b0e60f2b8689 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 12:35:27 +0200 Subject: [PATCH 01/24] Add pip and apt cache optimization, change golang to versions 1.20. --- docker/worker/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index f3387b1ad..eb173e2bf 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -3,9 +3,11 @@ FROM ubuntu:22.04 # Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable]" ARG PPA_TRACK=stable +ENV PIP_NO_CACHE_DIR=1 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -y upgrade -RUN apt-get -y install \ +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get update && apt-get -y upgrade && + apt-get -y install \ apt-transport-https \ apt-utils \ automake \ @@ -14,7 +16,7 @@ RUN apt-get -y install \ dh-autoreconf \ gcc \ git \ - golang \ + golang-1.20 \ gpg \ go-bindata \ john \ @@ -63,7 +65,8 @@ RUN pip3 install dfDewey RUN curl -sS https://keyserver.ubuntu.com/pks/lookup?op=get\&search=0x3ed1eaece81894b171d7da5b5e80511b10c598b8 | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/gift.gpg #RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x5e80511b10c598b8 \ RUN add-apt-repository -y ppa:gift/$PPA_TRACK -RUN apt-get update && apt-get -y install \ +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get update && apt-get -y install \ bulk-extractor \ dfimagetools-tools \ docker-explorer-tools \ From 3547100b5ce86e9138ea0a998367e4018639cf98 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 13:28:35 +0200 Subject: [PATCH 02/24] Move to one line. --- docker/worker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index eb173e2bf..888fd4152 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -6,8 +6,7 @@ ARG PPA_TRACK=stable ENV PIP_NO_CACHE_DIR=1 ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt \ - apt-get update && apt-get -y upgrade && - apt-get -y install \ + apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ automake \ From d2ea67473bd64239b9e39aece1c949317df5d791 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 13:40:04 +0200 Subject: [PATCH 03/24] pip and apt cache cleaning --- docker/worker/Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 888fd4152..8c64abe8a 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -5,8 +5,7 @@ ARG PPA_TRACK=stable ENV PIP_NO_CACHE_DIR=1 ENV DEBIAN_FRONTEND=noninteractive -RUN --mount=type=cache,target=/var/cache/apt \ - apt-get update && apt-get -y upgrade && apt-get -y install \ +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ automake \ @@ -40,14 +39,14 @@ RUN --mount=type=cache,target=/var/cache/apt \ && rm -rf /var/lib/apt/lists/* ADD requirements.txt /tmp/ -RUN cd /tmp/ && pip3 install -r requirements.txt +RUN cd /tmp/ && pip3 install --no-cache-dir -r requirements.txt -RUN pip3 install pip --upgrade -RUN pip3 install requests --upgrade -RUN pip3 install urllib3 cryptography --upgrade +RUN pip3 install --no-cache-dir pip --upgrade +RUN pip3 install --no-cache-dir requests --upgrade +RUN pip3 install --no-cache-dir urllib3 cryptography --upgrade # Install third-party worker dependencies -RUN pip3 install dfDewey +RUN pip3 install --no-cache-dir dfDewey # TODO(hacktobeer) uncomment when protobuf lib dependency if fixed upstream # RUN pip3 install pyhindsight @@ -64,8 +63,7 @@ RUN pip3 install dfDewey RUN curl -sS https://keyserver.ubuntu.com/pks/lookup?op=get\&search=0x3ed1eaece81894b171d7da5b5e80511b10c598b8 | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/gift.gpg #RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x5e80511b10c598b8 \ RUN add-apt-repository -y ppa:gift/$PPA_TRACK -RUN --mount=type=cache,target=/var/cache/apt \ - apt-get update && apt-get -y install \ +RUN apt-get update && apt-get -y install \ bulk-extractor \ dfimagetools-tools \ docker-explorer-tools \ @@ -82,10 +80,12 @@ RUN --mount=type=cache,target=/var/cache/apt \ sleuthkit \ --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 +RUN apt-get clean + RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia -RUN pip3 install impacket --no-deps +RUN pip3 install --no-cache-dir impacket --no-deps RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ From bfa939a72cf3e46a550201f7449a1088405a46f4 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 13:48:46 +0200 Subject: [PATCH 04/24] Ad golang symlink to compiler. --- docker/worker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 8c64abe8a..c759a0e7e 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -38,6 +38,9 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ wget \ && rm -rf /var/lib/apt/lists/* +# Add a symlink as we install the non-default golang compiler +RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go + ADD requirements.txt /tmp/ RUN cd /tmp/ && pip3 install --no-cache-dir -r requirements.txt From 9d077313800dedc7f910a9117e96cd3c992091d8 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 14:34:49 +0200 Subject: [PATCH 05/24] Cleanup fraken and turbinia build folders --- docker/worker/Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index c759a0e7e..c53c75290 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ wget \ && rm -rf /var/lib/apt/lists/* -# Add a symlink as we install the non-default golang compiler +# Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go ADD requirements.txt /tmp/ @@ -63,8 +63,6 @@ RUN pip3 install --no-cache-dir dfDewey # Plaso # Sleuthkit -RUN curl -sS https://keyserver.ubuntu.com/pks/lookup?op=get\&search=0x3ed1eaece81894b171d7da5b5e80511b10c598b8 | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/gift.gpg -#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x5e80511b10c598b8 \ RUN add-apt-repository -y ppa:gift/$PPA_TRACK RUN apt-get update && apt-get -y install \ bulk-extractor \ @@ -83,6 +81,7 @@ RUN apt-get update && apt-get -y install \ sleuthkit \ --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 +# Cleanup apt files RUN apt-get clean RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia @@ -93,6 +92,8 @@ RUN pip3 install --no-cache-dir impacket --no-deps RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ && sudo chown -R turbinia:turbinia /opt/signature-base +# Cleanup folder as Fraken only uses rules with .yar or .yara extensions +RUN find /opt/signature-base/ -type f ! -iname "*.yar*" -delete COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ @@ -103,6 +104,9 @@ RUN cd /opt/fraken/yara && ./bootstrap.sh \ && ./configure \ && make && sudo make install && sudo ldconfig RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken +# Cleanup fraken build folder and yara library source +RUN find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara + RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ @@ -130,6 +134,8 @@ ADD . /tmp/ # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi RUN cd /tmp/ && python3 setup.py install +# Cleanup /tmp folder (including dot files) +RUN find /tmp/ -mindepth 1 -delete COPY docker/worker/start.sh /home/turbinia/start.sh RUN chmod +rwx /home/turbinia/start.sh From 22f1e1fa79d139966f63e535d72335a7ddbb3779 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 15:03:47 +0200 Subject: [PATCH 06/24] move apt clean --- docker/worker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index c53c75290..795052760 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -5,6 +5,7 @@ ARG PPA_TRACK=stable ENV PIP_NO_CACHE_DIR=1 ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ @@ -36,7 +37,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ sudo \ testdisk \ wget \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/lib/apt/lists/* # Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go @@ -79,10 +80,9 @@ RUN apt-get update && apt-get -y install \ python3-dfvfs \ python3-plaso \ sleuthkit \ - --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 + --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -# Cleanup apt files -RUN apt-get clean RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia From 519f01a07ed27ea49a8fefe2a6c47a55bb9c6367 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 15:19:53 +0200 Subject: [PATCH 07/24] merge build layers --- docker/worker/Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 795052760..122489f53 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -37,7 +37,8 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ sudo \ testdisk \ wget \ - && apt-get clean && rm -rf /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + # Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go @@ -81,7 +82,8 @@ RUN apt-get update && apt-get -y install \ python3-plaso \ sleuthkit \ --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 \ - && apt-get clean && rm -rf /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia @@ -91,21 +93,19 @@ RUN pip3 install --no-cache-dir impacket --no-deps RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ - && sudo chown -R turbinia:turbinia /opt/signature-base -# Cleanup folder as Fraken only uses rules with .yar or .yara extensions -RUN find /opt/signature-base/ -type f ! -iname "*.yar*" -delete + && sudo chown -R turbinia:turbinia /opt/signature-base \ + && find /opt/signature-base/ -type f ! -iname "*.yar*" -delete COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ RUN mkdir -p /opt/fraken/yara && chown -R turbinia:turbinia /opt/fraken COPY --chown=turbinia:turbinia tools/fraken/* /opt/fraken/ -RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 -RUN cd /opt/fraken/yara && ./bootstrap.sh \ +RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 \ + && cd /opt/fraken/yara && ./bootstrap.sh \ && ./configure \ - && make && sudo make install && sudo ldconfig -RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken -# Cleanup fraken build folder and yara library source -RUN find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara + && make && sudo make install && sudo ldconfig \ + && cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken \ + && find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ @@ -132,10 +132,10 @@ RUN sudo /tmp/container-explorer-setup.sh install ADD . /tmp/ # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone -RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi -RUN cd /tmp/ && python3 setup.py install -# Cleanup /tmp folder (including dot files) -RUN find /tmp/ -mindepth 1 -delete +# install turbinia and cleanup /tmp +RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ + && cd /tmp/ && python3 setup.py install \ + && find /tmp/ -mindepth 1 -delete COPY docker/worker/start.sh /home/turbinia/start.sh RUN chmod +rwx /home/turbinia/start.sh From bfb682dca96defd3351d13eceeddcaee5177d01c Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 16:12:06 +0200 Subject: [PATCH 08/24] static fraken --- docker/worker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 122489f53..cea833581 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -104,9 +104,10 @@ RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https:// && cd /opt/fraken/yara && ./bootstrap.sh \ && ./configure \ && make && sudo make install && sudo ldconfig \ - && cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken \ + && cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static -extldflags=-ljemalloc" -o fraken \ && find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara +RUN ls -al /opt/fraken RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ @@ -122,8 +123,7 @@ RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessle RUN cp /home/turbinia/password.lst /root/ # Copy Kubernetes support tool to home folder -COPY k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py -RUN chown turbinia:turbinia /home/turbinia/check-lockfile.py +COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py # Install container-explorer RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh From 885e97314339dbe2e0875123aa354007d970617f Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 16:39:27 +0200 Subject: [PATCH 09/24] move fraken to multi-stage build --- docker/worker/Dockerfile | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index cea833581..b1edea9a9 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -1,4 +1,29 @@ FROM ubuntu:22.04 +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ + automake \ + curl \ + dh-autoreconf \ + gcc \ + golang-1.20 \ + go-bindata \ + libjemalloc-dev \ + libprotobuf-c-dev \ + libssl-dev \ + libtool \ + make \ + pkg-config \ + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + +RUN mkdir -p /opt/fraken/yara +RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 \ + && cd /opt/fraken/yara && ./bootstrap.sh \ + && ./configure \ + && make && sudo make install && sudo ldconfig \ + && cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static -extldflags=-ljemalloc" -o fraken \ + && find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara + +FROM ubuntu:22.04 +ENV DEBIAN_FRONTEND=noninteractive # Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable]" ARG PPA_TRACK=stable @@ -98,16 +123,9 @@ RUN cd /opt \ COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ -RUN mkdir -p /opt/fraken/yara && chown -R turbinia:turbinia /opt/fraken -COPY --chown=turbinia:turbinia tools/fraken/* /opt/fraken/ -RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 \ - && cd /opt/fraken/yara && ./bootstrap.sh \ - && ./configure \ - && make && sudo make install && sudo ldconfig \ - && cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static -extldflags=-ljemalloc" -o fraken \ - && find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara - -RUN ls -al /opt/fraken +# copy fraken in +RUN mkdir -p /opt/fraken +COPY --chown=turbinia:turbinia --from=0 /opt/fraken/fraken /opt/fraken/fraken RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ From 7d75c59b9ef0fb6af93f3239c9d9d5cbb55e2b4e Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 16:40:42 +0200 Subject: [PATCH 10/24] remove fraken deps from main build stage --- docker/worker/Dockerfile | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index b1edea9a9..dd5fc85f1 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -34,29 +34,17 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ - automake \ ca-certificates \ curl \ - dh-autoreconf \ - gcc \ git \ - golang-1.20 \ - gpg \ - go-bindata \ john \ john-data \ hashcat \ hashcat-data \ - libjemalloc-dev \ libleveldb1d \ libleveldb-dev \ - libprotobuf-c-dev \ - libssl-dev \ libterm-readline-gnu-perl \ - libtool \ lvm2 \ - make \ - pkg-config \ python3-pip \ software-properties-common \ sudo \ From d06b8d528b3f706efc6c66a8c2c74e3c2538f166 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 17:05:24 +0200 Subject: [PATCH 11/24] multi-stage fraken improvements --- docker/worker/Dockerfile | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index dd5fc85f1..71fc5c853 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -1,3 +1,5 @@ +# Multi-stage build +# Build 0 - fraken FROM ubuntu:22.04 RUN apt-get update && apt-get -y upgrade && apt-get -y install \ automake \ @@ -12,16 +14,17 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ libtool \ make \ pkg-config \ - && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + sudo +# Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) +RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go RUN mkdir -p /opt/fraken/yara -RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 \ - && cd /opt/fraken/yara && ./bootstrap.sh \ - && ./configure \ - && make && sudo make install && sudo ldconfig \ - && cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static -extldflags=-ljemalloc" -o fraken \ - && find /opt/fraken/ -type f ! -iname "fraken" -delete && rm -fr /opt/fraken/yara +RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 +RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig +COPY tools/fraken/* /opt/fraken/ +RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static -extldflags=-ljemalloc" -o fraken +# Build 1 - Turbinia Worker FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive @@ -53,9 +56,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* -# Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) -RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go - ADD requirements.txt /tmp/ RUN cd /tmp/ && pip3 install --no-cache-dir -r requirements.txt @@ -97,8 +97,6 @@ RUN apt-get update && apt-get -y install \ --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* - - RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia From 5ab71d541b9f2d898fb31a6587a954ab05bec7ee Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 27 Jul 2023 18:08:24 +0200 Subject: [PATCH 12/24] static fraken --- docker/worker/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 71fc5c853..83f0f82d4 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -1,6 +1,6 @@ # Multi-stage build # Build 0 - fraken -FROM ubuntu:22.04 +FROM ubuntu:22.04 AS fraken-builder RUN apt-get update && apt-get -y upgrade && apt-get -y install \ automake \ curl \ @@ -15,6 +15,8 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ make \ pkg-config \ sudo + # musl-dev \ + # musl-tools # Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go @@ -22,10 +24,12 @@ RUN mkdir -p /opt/fraken/yara RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig COPY tools/fraken/* /opt/fraken/ -RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static -extldflags=-ljemalloc" -o fraken +#RUN cd /opt/fraken && CGO_ENABLED=1 CC=musl-gcc go build --ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken +RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken +RUN ldd /opt/fraken/fraken # Build 1 - Turbinia Worker -FROM ubuntu:22.04 +FROM ubuntu:22.04 AS worker-builder ENV DEBIAN_FRONTEND=noninteractive # Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable]" @@ -111,7 +115,7 @@ COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ # copy fraken in RUN mkdir -p /opt/fraken -COPY --chown=turbinia:turbinia --from=0 /opt/fraken/fraken /opt/fraken/fraken +COPY --chown=turbinia:turbinia --from=fraken-builder /opt/fraken/fraken /opt/fraken/fraken RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ From e615d5db09d40112a8734dba56c1258e5ddf91c8 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 10:01:46 +0200 Subject: [PATCH 13/24] Add latest golang version manually. --- docker/worker/Dockerfile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 83f0f82d4..7f44904a3 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -1,4 +1,6 @@ # Multi-stage build +# Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable] GOVERSION=[1.20.6|1.18|?]" + # Build 0 - fraken FROM ubuntu:22.04 AS fraken-builder RUN apt-get update && apt-get -y upgrade && apt-get -y install \ @@ -6,8 +8,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ curl \ dh-autoreconf \ gcc \ - golang-1.20 \ - go-bindata \ libjemalloc-dev \ libprotobuf-c-dev \ libssl-dev \ @@ -18,21 +18,26 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ # musl-dev \ # musl-tools # Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) -RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go +#RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go + +# Install latest golang as ubuntu base image provides outdated version. +ARG GOVERSION=1.20.6 +RUN curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz +RUN tar -C /usr/local -xvf go$GOVERSION.linux-amd64.tar.gz +ENV PATH="$PATH:/usr/local/go/bin" RUN mkdir -p /opt/fraken/yara RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig COPY tools/fraken/* /opt/fraken/ +# TODO(rbdebere): Rework to use musl-gcc for full static build! #RUN cd /opt/fraken && CGO_ENABLED=1 CC=musl-gcc go build --ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken -RUN ldd /opt/fraken/fraken # Build 1 - Turbinia Worker FROM ubuntu:22.04 AS worker-builder ENV DEBIAN_FRONTEND=noninteractive -# Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable]" ARG PPA_TRACK=stable ENV PIP_NO_CACHE_DIR=1 From d7b5e05e0f549b23d84daec5a9da44182f923cdd Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 10:09:47 +0200 Subject: [PATCH 14/24] cleanup --- docker/worker/Dockerfile | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 7f44904a3..d5f90aae5 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -14,11 +14,11 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ libtool \ make \ pkg-config \ - sudo + sudo \ # musl-dev \ - # musl-tools -# Add a symlink as we install the non-default golang 1.20 compiler (latest for 22.04) -#RUN ln -s /usr/lib/go-1.20/bin/go /usr/bin/go + # musl-tools \ + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + # Install latest golang as ubuntu base image provides outdated version. ARG GOVERSION=1.20.6 @@ -64,7 +64,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ wget \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* - ADD requirements.txt /tmp/ RUN cd /tmp/ && pip3 install --no-cache-dir -r requirements.txt @@ -74,8 +73,8 @@ RUN pip3 install --no-cache-dir urllib3 cryptography --upgrade # Install third-party worker dependencies RUN pip3 install --no-cache-dir dfDewey -# TODO(hacktobeer) uncomment when protobuf lib dependency if fixed upstream -# RUN pip3 install pyhindsight +RUN pip3 install --no-cache-dir pyhindsight +RUN pip3 install --no-cache-dir impacket --no-deps # Install various packages from the GIFT PPA # bulkextractor @@ -103,22 +102,21 @@ RUN apt-get update && apt-get -y install \ python3-dfvfs \ python3-plaso \ sleuthkit \ - --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 \ +# --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia -RUN pip3 install --no-cache-dir impacket --no-deps +# RUN pip3 install --no-cache-dir impacket --no-deps +# Install fraken and yara rules. RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ && sudo chown -R turbinia:turbinia /opt/signature-base \ && find /opt/signature-base/ -type f ! -iname "*.yar*" -delete - COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ - -# copy fraken in +# Install fraken binary from multi-stage build RUN mkdir -p /opt/fraken COPY --chown=turbinia:turbinia --from=fraken-builder /opt/fraken/fraken /opt/fraken/fraken @@ -129,7 +127,7 @@ RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia && chown -R turbinia:turbinia /var/log/turbinia/ \ && mkdir -p /home/turbinia && chown -R turbinia:turbinia /home/turbinia -# Get a decent password list +# Get a decent password list for john/hashcat RUN cd /home/turbinia && echo "" > password.lst RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/285474cf9bff85f3323c5a1ae436f78acd1cb62c/Passwords/UserPassCombo-Jay.txt >> password.lst RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/Common-Credentials/10-million-password-list-top-1000000.txt >> password.lst From 2727625706ceb66768c91644c2f3cfc8f7527fe4 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 10:59:36 +0200 Subject: [PATCH 15/24] more cleanup --- docker/worker/Dockerfile | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index d5f90aae5..743a20317 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -23,26 +23,24 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ # Install latest golang as ubuntu base image provides outdated version. ARG GOVERSION=1.20.6 RUN curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz -RUN tar -C /usr/local -xvf go$GOVERSION.linux-amd64.tar.gz +RUN tar -C /usr/local -xf go$GOVERSION.linux-amd64.tar.gz ENV PATH="$PATH:/usr/local/go/bin" RUN mkdir -p /opt/fraken/yara RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig COPY tools/fraken/* /opt/fraken/ -# TODO(rbdebere): Rework to use musl-gcc for full static build! +# TODO(rbdebeer): Rework to use musl-gcc for full static build! #RUN cd /opt/fraken && CGO_ENABLED=1 CC=musl-gcc go build --ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken # Build 1 - Turbinia Worker FROM ubuntu:22.04 AS worker-builder ENV DEBIAN_FRONTEND=noninteractive +ENV PIP_NO_CACHE_DIR=1 ARG PPA_TRACK=stable -ENV PIP_NO_CACHE_DIR=1 -ENV DEBIAN_FRONTEND=noninteractive - RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ @@ -102,24 +100,28 @@ RUN apt-get update && apt-get -y install \ python3-dfvfs \ python3-plaso \ sleuthkit \ -# --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 \ - && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia # RUN pip3 install --no-cache-dir impacket --no-deps -# Install fraken and yara rules. +# Install yara rules and fraken. RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ && sudo chown -R turbinia:turbinia /opt/signature-base \ - && find /opt/signature-base/ -type f ! -iname "*.yar*" -delete + && find /opt/signature/base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ # Install fraken binary from multi-stage build RUN mkdir -p /opt/fraken COPY --chown=turbinia:turbinia --from=fraken-builder /opt/fraken/fraken /opt/fraken/fraken +# Install container-explorer +RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh +RUN chmod +x /tmp/container-explorer-setup.sh +RUN sudo /tmp/container-explorer-setup.sh install + RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ && mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \ @@ -136,11 +138,6 @@ RUN cp /home/turbinia/password.lst /root/ # Copy Kubernetes support tool to home folder COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py -# Install container-explorer -RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh -RUN chmod +x /tmp/container-explorer-setup.sh -RUN sudo /tmp/container-explorer-setup.sh install - ADD . /tmp/ # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone # install turbinia and cleanup /tmp From 73f90c3470c97c5f08f57aa427e91c23449b9939 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 11:19:38 +0200 Subject: [PATCH 16/24] typo --- docker/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 743a20317..751ec6d60 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -111,7 +111,7 @@ RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ && sudo chown -R turbinia:turbinia /opt/signature-base \ - && find /opt/signature/base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete + && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ # Install fraken binary from multi-stage build RUN mkdir -p /opt/fraken From ea3b061d45e78684d8cc4b3e6dc57a7ca1dbc367 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 14:57:12 +0200 Subject: [PATCH 17/24] More comments and cleanup --- docker/worker/Dockerfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 751ec6d60..c885a7c10 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -15,8 +15,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ make \ pkg-config \ sudo \ - # musl-dev \ - # musl-tools \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* @@ -26,6 +24,7 @@ RUN curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz RUN tar -C /usr/local -xf go$GOVERSION.linux-amd64.tar.gz ENV PATH="$PATH:/usr/local/go/bin" +# Fetch and compile libyara and fraken RUN mkdir -p /opt/fraken/yara RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig @@ -63,16 +62,19 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* ADD requirements.txt /tmp/ -RUN cd /tmp/ && pip3 install --no-cache-dir -r requirements.txt +RUN cd /tmp/ && pip3 install -r requirements.txt -RUN pip3 install --no-cache-dir pip --upgrade -RUN pip3 install --no-cache-dir requests --upgrade -RUN pip3 install --no-cache-dir urllib3 cryptography --upgrade +RUN pip3 install pip --upgrade +RUN pip3 install requests --upgrade +RUN pip3 install urllib3 cryptography --upgrade -# Install third-party worker dependencies -RUN pip3 install --no-cache-dir dfDewey -RUN pip3 install --no-cache-dir pyhindsight -RUN pip3 install --no-cache-dir impacket --no-deps +# Install third-party dependencies +# dfwdewey +# pyhindsight +# impacket +RUN pip3 install dfDewey +RUN pip3 install pyhindsight +RUN pip3 install impacket --no-deps # Install various packages from the GIFT PPA # bulkextractor @@ -102,12 +104,11 @@ RUN apt-get update && apt-get -y install \ sleuthkit \ && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* +# Add turbinia user to system and sudoers RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia -# RUN pip3 install --no-cache-dir impacket --no-deps - -# Install yara rules and fraken. +# Install yara rules and fraken binary. RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ && sudo chown -R turbinia:turbinia /opt/signature-base \ @@ -122,6 +123,7 @@ RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/g RUN chmod +x /tmp/container-explorer-setup.sh RUN sudo /tmp/container-explorer-setup.sh install +# Setup turbinia user folders and permissions RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ && mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \ @@ -138,9 +140,10 @@ RUN cp /home/turbinia/password.lst /root/ # Copy Kubernetes support tool to home folder COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py -ADD . /tmp/ +# Install Turbinia # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone # install turbinia and cleanup /tmp +ADD . /tmp/ RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ && cd /tmp/ && python3 setup.py install \ && find /tmp/ -mindepth 1 -delete From 8459a1179aa8e961a70f537c4e5c3a94f2672934 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 15:29:47 +0200 Subject: [PATCH 18/24] Remove Turbinia .git folder as well to safe space --- docker/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index c885a7c10..797e551d3 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -146,7 +146,7 @@ COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check- ADD . /tmp/ RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ && cd /tmp/ && python3 setup.py install \ - && find /tmp/ -mindepth 1 -delete + && rm -fr /tmp/.git COPY docker/worker/start.sh /home/turbinia/start.sh RUN chmod +rwx /home/turbinia/start.sh From 4c22b61e83986c7956cd369f5282180359b25fd3 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 15:49:20 +0200 Subject: [PATCH 19/24] Re-add .git as our setup and tests in github actions rely on it --- docker/worker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 797e551d3..b58fecbae 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -146,7 +146,6 @@ COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check- ADD . /tmp/ RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ && cd /tmp/ && python3 setup.py install \ - && rm -fr /tmp/.git COPY docker/worker/start.sh /home/turbinia/start.sh RUN chmod +rwx /home/turbinia/start.sh From 1db00f895db567747bb75d7b8b147fcd58ebdd2d Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Fri, 28 Jul 2023 15:58:12 +0200 Subject: [PATCH 20/24] ugh --- docker/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index b58fecbae..b4731cb5f 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -145,7 +145,7 @@ COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check- # install turbinia and cleanup /tmp ADD . /tmp/ RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ - && cd /tmp/ && python3 setup.py install \ + && cd /tmp/ && python3 setup.py install COPY docker/worker/start.sh /home/turbinia/start.sh RUN chmod +rwx /home/turbinia/start.sh From 458142c1682ab559abcad96587052c1a0cc0bc4e Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Mon, 31 Jul 2023 10:57:39 +0200 Subject: [PATCH 21/24] Add jemalloc mem lib again/ --- docker/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index b4731cb5f..5cd4877eb 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -31,7 +31,7 @@ RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make in COPY tools/fraken/* /opt/fraken/ # TODO(rbdebeer): Rework to use musl-gcc for full static build! #RUN cd /opt/fraken && CGO_ENABLED=1 CC=musl-gcc go build --ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken -RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken +RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc -extldflags=-static" -tags yara_static -o fraken # Build 1 - Turbinia Worker FROM ubuntu:22.04 AS worker-builder From 9efcb9019b7cef8965acb299bc8de3242b6fdfe3 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Mon, 31 Jul 2023 20:42:25 +0200 Subject: [PATCH 22/24] rewrite fraken build stage --- docker/worker/Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 5cd4877eb..d7d43d6c1 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -26,12 +26,10 @@ ENV PATH="$PATH:/usr/local/go/bin" # Fetch and compile libyara and fraken RUN mkdir -p /opt/fraken/yara -RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 -RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig +RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 +RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig COPY tools/fraken/* /opt/fraken/ -# TODO(rbdebeer): Rework to use musl-gcc for full static build! -#RUN cd /opt/fraken && CGO_ENABLED=1 CC=musl-gcc go build --ldflags="-linkmode=external -extldflags=-static" -tags yara_static -o fraken -RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc -extldflags=-static" -tags yara_static -o fraken +RUN cd /opt/fraken && go build -a -v -ldflags="-linkmode=external -extldflags=-ljemalloc -extldflags=-static" -tags yara_static -o fraken # Build 1 - Turbinia Worker FROM ubuntu:22.04 AS worker-builder From 4b67f6d941991194ecb5805fcf7e66f2e40a527b Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Tue, 1 Aug 2023 11:18:19 +0200 Subject: [PATCH 23/24] Alpine musl static build --- docker/worker/Dockerfile | 68 +++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index d7d43d6c1..67f8e5369 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -2,34 +2,51 @@ # Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable] GOVERSION=[1.20.6|1.18|?]" # Build 0 - fraken -FROM ubuntu:22.04 AS fraken-builder -RUN apt-get update && apt-get -y upgrade && apt-get -y install \ - automake \ - curl \ - dh-autoreconf \ - gcc \ - libjemalloc-dev \ - libprotobuf-c-dev \ - libssl-dev \ - libtool \ - make \ - pkg-config \ - sudo \ - && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* - - -# Install latest golang as ubuntu base image provides outdated version. -ARG GOVERSION=1.20.6 -RUN curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz -RUN tar -C /usr/local -xf go$GOVERSION.linux-amd64.tar.gz -ENV PATH="$PATH:/usr/local/go/bin" - -# Fetch and compile libyara and fraken +FROM golang:alpine AS fraken-builder +RUN apk add --no-cache -t .build-deps \ + autoconf \ + automake \ + bison \ + build-base \ + curl \ + file \ + file-dev \ + flex \ + git \ + jansson \ + jansson-dev \ + jansson-static \ + libc-dev \ + libmagic \ + libmagic-static \ + libtool \ + linux-headers \ + openssl \ + openssl-dev \ + openssl-libs-static \ + py3-setuptools \ + python3 \ + python3-dev \ + sudo + +RUN set -x \ + && echo "Compiling Yara from source..." + +# Fetch and compile libyara RUN mkdir -p /opt/fraken/yara +WORKDIR /opt/fraken/yara RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 -RUN cd /opt/fraken/yara && ./bootstrap.sh && ./configure && make && sudo make install && sudo ldconfig +RUN ./bootstrap.sh +RUN sync +RUN ./configure --with-crypto \ + --enable-magic \ + --enable-cuckoo +RUN make +RUN sudo make install + +# Compile fraken statically COPY tools/fraken/* /opt/fraken/ -RUN cd /opt/fraken && go build -a -v -ldflags="-linkmode=external -extldflags=-ljemalloc -extldflags=-static" -tags yara_static -o fraken +RUN cd /opt/fraken && GOOS=linux GOARCH=amd64 go build -a -v -ldflags="-linkmode=external -extldflags=-static" -installsuffix netgo -tags yara_static,osusergo,netgo -o fraken # Build 1 - Turbinia Worker FROM ubuntu:22.04 AS worker-builder @@ -83,7 +100,6 @@ RUN pip3 install impacket --no-deps # libluksde-tools # Plaso # Sleuthkit - RUN add-apt-repository -y ppa:gift/$PPA_TRACK RUN apt-get update && apt-get -y install \ bulk-extractor \ From 3b310c5fa5f14d27a2a57ac5a9e4e623d3af78c9 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Tue, 1 Aug 2023 12:07:15 +0200 Subject: [PATCH 24/24] Small Dockerfile server updates. --- docker/server/Dockerfile | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index b032b7604..5a9913e89 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,8 +1,8 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -y upgrade -RUN apt-get -y install \ +ENV PIP_NO_CACHE_DIR=1 +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ ca-certificates \ @@ -11,19 +11,17 @@ RUN apt-get -y install \ python3-pip \ software-properties-common \ sudo \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* -RUN pip3 install pip --upgrade -RUN pip3 install urllib3 cryptography requests --upgrade +RUN pip3 install pip --upgrade \ + && pip3 install urllib3 cryptography requests --upgrade -ADD requirements.txt /tmp/ +ADD . /tmp/ RUN cd /tmp/ && pip3 install -r requirements.txt -ADD . /tmp/ # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone -RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi - -RUN cd /tmp/ && python3 setup.py install +RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ + && cd /tmp/ && python3 setup.py install RUN useradd -r -s /bin/nologin -u 999 turbinia