From 517c61e3a650d3afaaddce7f9e5a40d932fa7340 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 8 Aug 2024 18:11:51 +0200 Subject: [PATCH 1/2] Add api-server debugging --- .vscode/launch.json | 16 ++++++++++++++-- docker/api_server/Dockerfile | 6 ++++++ skaffold.yaml | 26 ++++++++++++++++++++++---- turbinia/api/api_server.py | 11 ++++++++++- turbinia/debug.py | 8 ++++---- turbinia/turbiniactl.py | 1 + 6 files changed, 57 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 380926b8d..9f3c6f9da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "language": "Python", "debugPort": 10000, "podSelector": { - "app.kubernetes.io/component": "worker" + "app.kubernetes.io/component": "worker" }, "localRoot": "${workspaceFolder}", "remoteRoot": "/home/turbinia" @@ -20,7 +20,19 @@ "language": "Python", "debugPort": 20000, "podSelector": { - "app.kubernetes.io/component": "server" + "app.kubernetes.io/component": "server" + }, + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/turbinia" + }, + { + "name": "Attach to API Server", + "type": "cloudcode.kubernetes", + "request": "attach", + "language": "Python", + "debugPort": 30000, + "podSelector": { + "app.kubernetes.io/component": "api" }, "localRoot": "${workspaceFolder}", "remoteRoot": "/home/turbinia" diff --git a/docker/api_server/Dockerfile b/docker/api_server/Dockerfile index 5899dc800..337ee5e52 100644 --- a/docker/api_server/Dockerfile +++ b/docker/api_server/Dockerfile @@ -46,6 +46,12 @@ RUN chmod +rwx /home/turbinia/start.sh # Install Turbinia package -- will skip dependencies if installed RUN poetry install --no-interaction --no-ansi +# Create debug and env variables +ARG TURBINIA_DEBUG +ARG TURBINIA_DEBUG_PORT +ENV TURBINIA_DEBUG ${TURBINIA_DEBUG:-0} +ENV TURBINIA_DEBUG_PORT ${TURBINIA_DEBUG_PORT:-30000} + CMD ["/home/turbinia/start.sh"] # Expose Prometheus and API endpoints. EXPOSE 9200/tcp diff --git a/skaffold.yaml b/skaffold.yaml index d108be8ed..e28b6b762 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -20,7 +20,7 @@ build: # infer: # - turbinia/** - # Uncomment below when developing on the Turbinia Server + # # Uncomment below when developing on the Turbinia Server - image: turbinia-server docker: dockerfile: docker/server/Dockerfile @@ -31,6 +31,18 @@ build: sync: infer: - turbinia/** + + # # Uncomment below when developing on the Turbinia Server + # - image: turbinia-api-server + # docker: + # dockerfile: docker/api_server/Dockerfile + # buildArgs: + # TURBINIA_DEBUG: 1 + # TURBINIA_DEBUG_PORT: 30000 + # sync: + # infer: + # - turbinia/** + deploy: statusCheckDeadlineSeconds: 90 helm: @@ -42,15 +54,19 @@ deploy: # remoteChart: turbinia ## Uncomment below if using local helm charts chartPath: ./charts/turbinia + skipBuildDependencies: true setValues: versioncheck.enabled: False setValueTemplates: - ## Uncomment below if doing worker development + # # Uncomment below if doing worker development # worker.image.repository: "{{.IMAGE_REPO_turbinia_worker}}" # worker.image.tag: "{{.IMAGE_TAG_turbinia_worker}}@{{.IMAGE_DIGEST_turbinia_worker}}" - ## Uncomment below if doing server development + # Uncomment below if doing server development server.image.repository: "{{.IMAGE_REPO_turbinia_server}}" server.image.tag: "{{.IMAGE_TAG_turbinia_server}}@{{.IMAGE_DIGEST_turbinia_server}}" + # # Uncomment below if doing API server development + # api.image.repository: "{{.IMAGE_REPO_turbinia_api_server}}" + # api.image.tag: "{{.IMAGE_TAG_turbinia_api_server}}@{{.IMAGE_DIGEST_turbinia_api_server}}" portForward: - resourceType: deployment resourceName: dev-release-turbinia-api @@ -61,4 +77,6 @@ portForward: - resourceType: deployment resourceName: dev-release-turbinia-server port: 20000 # Server debug port - +- resourceType: deployment + resourceName: dev-release-turbinia-api + port: 30000 # API Server debug port diff --git a/turbinia/api/api_server.py b/turbinia/api/api_server.py index 63a0dff8c..f5cd1b268 100644 --- a/turbinia/api/api_server.py +++ b/turbinia/api/api_server.py @@ -16,6 +16,7 @@ import io import logging +from os import getenv import yaml import uvicorn @@ -97,9 +98,17 @@ def start(self, app_name: str): app_name (str): module:app string used by Uvicorn to start the HTTP server. """ + reload = False + workers = 4 + log_level = 'info' + if getenv('TURBINIA_DEBUG') == '1': + reload = True + workers = 0 + log_level = 'debug' + uvicorn.run( app_name, host=config.API_SERVER_ADDRESS, port=config.API_SERVER_PORT, - log_config=None, log_level='info', reload=False, workers=4) + log_config=None, log_level=log_level, reload=reload, workers=workers) if __name__ == '__main__': diff --git a/turbinia/debug.py b/turbinia/debug.py index 1d036a894..283850670 100644 --- a/turbinia/debug.py +++ b/turbinia/debug.py @@ -2,12 +2,12 @@ def initialize_debugmode_if_requested(): - if getenv('TURBINIA_DEBUG') == "1": + if getenv('TURBINIA_DEBUG') == '1': import debugpy debug_port = getenv('TURBINIA_DEBUG_PORT') - debugpy.listen(("0.0.0.0", int(debug_port))) - print('Debugger can now be attached on port {0:s}'.format(debug_port)) - if getenv('TURBINIA_HOTRELOAD') == "1": + debugpy.listen(('0.0.0.0', int(debug_port))) + print(f'Debugger can now be attached on port {debug_port}') + if getenv('TURBINIA_HOTRELOAD') == '1': import jurigged jurigged.watch('turbinia/') print('Code hot reloading enabled.') diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py index 97e079ac3..76d761134 100644 --- a/turbinia/turbiniactl.py +++ b/turbinia/turbiniactl.py @@ -196,6 +196,7 @@ def process_args(args): jobs_denylist=args.jobs_denylist, jobs_allowlist=args.jobs_allowlist) server.start() elif args.command == 'api_server': + initialize_debugmode_if_requested() # pylint: disable=import-outside-toplevel from turbinia.api.api_server import TurbiniaAPIServer api_server = TurbiniaAPIServer() From dd155ad27e6e0d0d68aeaddb2e2b6131c2ffac61 Mon Sep 17 00:00:00 2001 From: hacktobeer Date: Thu, 8 Aug 2024 18:18:04 +0200 Subject: [PATCH 2/2] Add documentation paragraph. --- docs/developer/develop-minikube.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/developer/develop-minikube.md b/docs/developer/develop-minikube.md index ece5bfe25..f6366d0ce 100644 --- a/docs/developer/develop-minikube.md +++ b/docs/developer/develop-minikube.md @@ -178,6 +178,9 @@ Try our Turbinia minikube development 101 Codelab [here](turbinia-codelab-analys * Remove the Google Cloud Code Extension from VSCode ### Troubleshooting and Tips +#### Debugging API Server +You can enable debugging by uncommenting the API Server `build` and `setValueTemplates` sections in the `skaffold.yaml` file. + #### Google Cloud Code tools not found (minikube, skaffold, kubectl) The extension will add the PATH automatically to your config. But if you have a different or custom shell configuration this may fail. Add the path manually to your PATH. * Linux: `$HOME/.cache/cloud-code/installer/google-cloud-sdk/bin`