Attempt to resolve poetry conflict

google · Nov 19, 2024 · 07ee232 · 07ee232
2 parents d84d2c0 + c778de7
commit 07ee232
Show file tree

Hide file tree

Showing 43 changed files with 1,510 additions and 1,225 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -2,7 +2,61 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Attach to Worker",
+            "name": "Attach debugpy Worker",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port":10000
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/home/turbinia"
+                }
+            ]
+        },
+        {
+            "name": "Attach debugpy Server",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port":20000
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/home/turbinia"
+                }
+            ]
+        },
+        {
+            "name": "Attach debugpy API Server",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port":30000
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/home/turbinia"
+                }
+            ]
+        },
+        {
+            "name": "Turbinia API Client",
+            "type": "debugpy",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "program": "turbinia/api/cli/turbinia_client/turbiniacli_tool.py",
+            "args": "${input:apiClientCommand}",
+            "console": "integratedTerminal",
+        },
+        {
+            "name": "Attach k8s Worker",
             "type": "cloudcode.kubernetes",
             "request": "attach",
             "language": "Python",
@@ -14,7 +68,7 @@
             "remoteRoot": "/home/turbinia"
         },
         {
-            "name": "Attach to Server",
+            "name": "Attach k8s Server",
             "type": "cloudcode.kubernetes",
             "request": "attach",
             "language": "Python",
@@ -26,7 +80,7 @@
             "remoteRoot": "/home/turbinia"
         },
         {
-            "name": "Attach to API Server",
+            "name": "Attach k8s API Server",
             "type": "cloudcode.kubernetes",
             "request": "attach",
             "language": "Python",
@@ -37,5 +91,13 @@
             "localRoot": "${workspaceFolder}",
             "remoteRoot": "/home/turbinia"
         }
-    ]
+    ],
+    "inputs": [
+    {
+      "id": "apiClientCommand",
+      "type": "promptString",
+      "description": "API Client Command",
+      "default": "config list"
+    }
+  ]
 }
diff --git a/docker/api_server/Dockerfile b/docker/api_server/Dockerfile
@@ -7,7 +7,7 @@ COPY web/. .
 RUN npm run build
 
 # Build Turbinia API Server, copy from build, and setup rest of requirements
-FROM ubuntu:22.04 as build-stage2
+FROM --platform=linux/amd64 ubuntu:22.04 as build-stage2
 
 ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 

diff --git a/docker/api_server/Dockerfile.dockerignore b/docker/api_server/Dockerfile.dockerignore
@@ -15,3 +15,6 @@ build
 __pycache__
 .cache
 turbinia.egg-info
+conf/
+redis-data/
+evidence/
diff --git a/docker/local/docker-compose.yml b/docker/local/docker-compose.yml
@@ -25,6 +25,8 @@ services:
             - LC_ALL=C.UTF-8
             - LANG=C.UTF-8
             - TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
+        ports:
+            - 127.0.0.1:20000:20000
 
     turbinia-api-server:
         #image: "turbinia-api-server-dev" # Use this for local development and comment out below line
@@ -41,6 +43,9 @@ services:
             - TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
         expose:
             - "8000"
+        ports:
+            - 127.0.0.1:30000:30000
+            - 127.0.0.1:8000:8000
 
     turbinia-worker:
         #image: "turbinia-worker-dev" # Use this for local development and comment out below line
@@ -56,6 +61,8 @@ services:
             - LC_ALL=C.UTF-8
             - LANG=C.UTF-8
             - TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
+        ports:
+            - 127.0.0.1:10000:10000
 
     # Uncomment below in case you want to run a second worker on the same host.
     #  turbinia-worker2:

diff --git a/docker/local/local-config.sed b/docker/local/local-config.sed
@@ -1,5 +1,5 @@
 s/CELERY_BROKER = .*/CELERY_BROKER = 'redis:\/\/redis'/g
-s/CELERY_BACKEND = .*/CELERY_BACKEND = 'redis:\/\/redis'/g
+s/CELERY_BACKEND = .*/CELERY_BACKEND = 'redis:\/\/redis\/1'/g
 s/REDIS_HOST = .*/REDIS_HOST = 'redis'/g
 s/PROMETHEUS_ENABLED = .*/PROMETHEUS_ENABLED = False/g
 s/STATE_MANAGER = .*/STATE_MANAGER = 'Redis'/g

diff --git a/docker/local/setup.sh b/docker/local/setup.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+mkdir redis-data
+mkdir evidence
+mkdir conf
+chmod 777 conf evidence redis-data
+sed -f docker/local/local-config.sed turbinia/config/turbinia_config_tmpl.py > conf/turbinia.conf
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive 
 ENV PIP_NO_CACHE_DIR=1

diff --git a/docker/server/Dockerfile.dockerignore b/docker/server/Dockerfile.dockerignore
@@ -16,3 +16,6 @@ charts
 __pycache__
 .cache
 turbinia.egg-info
+conf/
+redis-data/
+evidence/
diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 
 ARG PPA_TRACK=stable
@@ -57,9 +57,10 @@ RUN echo "Defaults secure_path=\"/home/turbinia/.venv/bin:/usr/local/sbin:/usr/l
 
 # Install yara rules and fraken binary.
 RUN cd /opt \
-    && git clone https://github.com/Neo23x0/signature-base.git \
+    && git clone https://github.com/Neo23x0/signature-base.git --depth=1\
     && sudo chown -R turbinia:turbinia /opt/signature-base \
-    && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
+    && sudo git config --global --add safe.directory /opt/signature-base \
+    && find /opt/signature-base -type f -not -path '*.git/*' -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
 COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
 RUN mkdir -p /opt/fraken && chown -R turbinia:turbinia /opt/fraken
 COPY --from=us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest --chown=turbinia:turbinia /bin/fraken /opt/fraken/fraken

diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile
@@ -1,5 +1,5 @@
 # Build 1 - Turbinia Worker
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1
 ARG PPA_TRACK=stable
@@ -52,9 +52,10 @@ RUN echo "Defaults secure_path=\"/home/turbinia/.venv/bin:/opt/fraken:/usr/local
 
 # Install yara rules and fraken binary.
 RUN cd /opt \
-    && git clone https://github.com/Neo23x0/signature-base.git \
+    && git clone https://github.com/Neo23x0/signature-base.git --depth=1\
     && sudo chown -R turbinia:turbinia /opt/signature-base \
-    && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
+    && sudo git config --global --add safe.directory /opt/signature-base \
+    && find /opt/signature-base -type f -not -path '*.git/*' -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
 COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
 RUN mkdir -p /opt/fraken && chown -R turbinia:turbinia /opt/fraken
 COPY --from=us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest --chown=turbinia:turbinia /bin/fraken /opt/fraken/fraken
@@ -112,8 +113,6 @@ ENV PATH="/home/turbinia/.venv/bin:$PATH" \
 
 # Copy the source directory to the container
 COPY --chown=turbinia:turbinia . /home/turbinia/
-COPY --chown=turbinia:turbinia docker/worker/start.sh /home/turbinia/start.sh
-RUN chmod +rwx /home/turbinia/start.sh
 
 # Install Turbinia package -- will skip dependencies if installed
 RUN poetry install --no-interaction --no-ansi -E worker
@@ -123,9 +122,11 @@ ARG TURBINIA_DEBUG
 ARG TURBINIA_HOTRELOAD
 ARG TURBINIA_DEBUG_PORT
 ENV TURBINIA_DEBUG ${TURBINIA_DEBUG:-0}
-ENV TURBINIA_HOTRELOAD ${TURBINIA_HOTRELOAD:-0}
 ENV TURBINIA_DEBUG_PORT ${TURBINIA_DEBUG_PORT:-10000}
 
-CMD ["/home/turbinia/start.sh"]
+CMD ["celery","--app","turbinia.app","worker","-l","INFO","--without-gossip","-E"]
+# Use below CMD instead to enable hot reloading of celery when code changes.
+# CMD ["watchmedo","auto-restart","--directory=./","--pattern=*.py","--recursive","--","celery","--app","turbinia.app","worker","-l","INFO","--without-gossip","-E"]
+
 # Expose Prometheus endpoint.
 EXPOSE 9200/tcp
diff --git a/docker/worker/Dockerfile.dockerignore b/docker/worker/Dockerfile.dockerignore
@@ -16,3 +16,6 @@ charts
 __pycache__
 .cache
 turbinia.egg-info
+conf/
+redis-data/
+evidence/
diff --git a/docker/worker/start.sh b/docker/worker/start.sh
diff --git a/docs/developer/turbinia-codelab-analyser.md b/docs/developer/turbinia-codelab-analyser.md
@@ -436,7 +436,7 @@ if __name__ == '__main__':
 
 Now let's install the test requirements and run this specific test from the root turbinia folder.
 ```
-$ pip install -f requirements-test.txt
+$ poetry install --with test
 $ PYTHONPATH=. python -m unittest turbinia.workers.analysis.os_info_test
 
 Using fallback source config Copy turbinia/config/turbinia_config_tmpl.py to ~/.turbiniarc or /etc/turbinia/turbinia.conf, edit, and re-run.