diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index b032b7604..5a9913e89 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,8 +1,8 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -y upgrade -RUN apt-get -y install \ +ENV PIP_NO_CACHE_DIR=1 +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ ca-certificates \ @@ -11,19 +11,17 @@ RUN apt-get -y install \ python3-pip \ software-properties-common \ sudo \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* -RUN pip3 install pip --upgrade -RUN pip3 install urllib3 cryptography requests --upgrade +RUN pip3 install pip --upgrade \ + && pip3 install urllib3 cryptography requests --upgrade -ADD requirements.txt /tmp/ +ADD . /tmp/ RUN cd /tmp/ && pip3 install -r requirements.txt -ADD . /tmp/ # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone -RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi - -RUN cd /tmp/ && python3 setup.py install +RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ + && cd /tmp/ && python3 setup.py install RUN useradd -r -s /bin/nologin -u 999 turbinia diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 0e1afdb98..67f8e5369 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -1,42 +1,80 @@ -FROM ubuntu:22.04 +# Multi-stage build +# Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable] GOVERSION=[1.20.6|1.18|?]" + +# Build 0 - fraken +FROM golang:alpine AS fraken-builder +RUN apk add --no-cache -t .build-deps \ + autoconf \ + automake \ + bison \ + build-base \ + curl \ + file \ + file-dev \ + flex \ + git \ + jansson \ + jansson-dev \ + jansson-static \ + libc-dev \ + libmagic \ + libmagic-static \ + libtool \ + linux-headers \ + openssl \ + openssl-dev \ + openssl-libs-static \ + py3-setuptools \ + python3 \ + python3-dev \ + sudo + +RUN set -x \ + && echo "Compiling Yara from source..." + +# Fetch and compile libyara +RUN mkdir -p /opt/fraken/yara +WORKDIR /opt/fraken/yara +RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 +RUN ./bootstrap.sh +RUN sync +RUN ./configure --with-crypto \ + --enable-magic \ + --enable-cuckoo +RUN make +RUN sudo make install + +# Compile fraken statically +COPY tools/fraken/* /opt/fraken/ +RUN cd /opt/fraken && GOOS=linux GOARCH=amd64 go build -a -v -ldflags="-linkmode=external -extldflags=-static" -installsuffix netgo -tags yara_static,osusergo,netgo -o fraken + +# Build 1 - Turbinia Worker +FROM ubuntu:22.04 AS worker-builder +ENV DEBIAN_FRONTEND=noninteractive +ENV PIP_NO_CACHE_DIR=1 -# Use: docker build --no-cache --build-arg PPA_TRACK="[staging|stable]" ARG PPA_TRACK=stable -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -y upgrade -RUN apt-get -y install \ +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ apt-transport-https \ apt-utils \ - automake \ ca-certificates \ curl \ - dh-autoreconf \ - gcc \ git \ - golang \ - gpg \ - go-bindata \ john \ john-data \ hashcat \ hashcat-data \ - libjemalloc-dev \ libleveldb1d \ libleveldb-dev \ - libprotobuf-c-dev \ - libssl-dev \ libterm-readline-gnu-perl \ - libtool \ lvm2 \ - make \ - pkg-config \ python3-pip \ software-properties-common \ sudo \ testdisk \ wget \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* ADD requirements.txt /tmp/ RUN cd /tmp/ && pip3 install -r requirements.txt @@ -45,10 +83,13 @@ RUN pip3 install pip --upgrade RUN pip3 install requests --upgrade RUN pip3 install urllib3 cryptography --upgrade -# Install third-party worker dependencies +# Install third-party dependencies +# dfwdewey +# pyhindsight +# impacket RUN pip3 install dfDewey -# TODO(hacktobeer) uncomment when protobuf lib dependency if fixed upstream -# RUN pip3 install pyhindsight +RUN pip3 install pyhindsight +RUN pip3 install impacket --no-deps # Install various packages from the GIFT PPA # bulkextractor @@ -59,7 +100,6 @@ RUN pip3 install dfDewey # libluksde-tools # Plaso # Sleuthkit - RUN add-apt-repository -y ppa:gift/$PPA_TRACK RUN apt-get update && apt-get -y install \ bulk-extractor \ @@ -76,27 +116,28 @@ RUN apt-get update && apt-get -y install \ python3-dfvfs \ python3-plaso \ sleuthkit \ - --option Acquire::ForceIPv4=true --option Acquire::Retries=100 --option Acquire::http::Timeout=60 + && apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* +# Add turbinia user to system and sudoers RUN useradd -r -s /bin/nologin -G disk,sudo -u 999 turbinia RUN echo "turbinia ALL = (root) NOPASSWD: ALL" > /etc/sudoers.d/turbinia -RUN pip3 install impacket --no-deps - +# Install yara rules and fraken binary. RUN cd /opt \ && git clone https://github.com/Neo23x0/signature-base.git \ - && sudo chown -R turbinia:turbinia /opt/signature-base - + && sudo chown -R turbinia:turbinia /opt/signature-base \ + && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete COPY turbinia/config/rules/*.yar /opt/signature-base/yara/ +# Install fraken binary from multi-stage build +RUN mkdir -p /opt/fraken +COPY --chown=turbinia:turbinia --from=fraken-builder /opt/fraken/fraken /opt/fraken/fraken -RUN mkdir -p /opt/fraken/yara && chown -R turbinia:turbinia /opt/fraken -COPY --chown=turbinia:turbinia tools/fraken/* /opt/fraken/ -RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')" | tar -xz --strip-components=1 -RUN cd /opt/fraken/yara && ./bootstrap.sh \ - && ./configure \ - && make && sudo make install && sudo ldconfig -RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken +# Install container-explorer +RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh +RUN chmod +x /tmp/container-explorer-setup.sh +RUN sudo /tmp/container-explorer-setup.sh install +# Setup turbinia user folders and permissions RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \ && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \ && mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \ @@ -104,25 +145,21 @@ RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia && chown -R turbinia:turbinia /var/log/turbinia/ \ && mkdir -p /home/turbinia && chown -R turbinia:turbinia /home/turbinia -# Get a decent password list +# Get a decent password list for john/hashcat RUN cd /home/turbinia && echo "" > password.lst RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/285474cf9bff85f3323c5a1ae436f78acd1cb62c/Passwords/UserPassCombo-Jay.txt >> password.lst RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/Common-Credentials/10-million-password-list-top-1000000.txt >> password.lst RUN cp /home/turbinia/password.lst /root/ # Copy Kubernetes support tool to home folder -COPY k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py -RUN chown turbinia:turbinia /home/turbinia/check-lockfile.py - -# Install container-explorer -RUN wget -O /tmp/container-explorer-setup.sh https://raw.githubusercontent.com/google/container-explorer/main/script/setup.sh -RUN chmod +x /tmp/container-explorer-setup.sh -RUN sudo /tmp/container-explorer-setup.sh install +COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py -ADD . /tmp/ +# Install Turbinia # unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone -RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi -RUN cd /tmp/ && python3 setup.py install +# install turbinia and cleanup /tmp +ADD . /tmp/ +RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \ + && cd /tmp/ && python3 setup.py install COPY docker/worker/start.sh /home/turbinia/start.sh RUN chmod +rwx /home/turbinia/start.sh