bugfix/DT-1770-add-localstack-into-docker-compose (#3006)

* add localstack dockerfile

* dont run the av on local

* fix broken ports on docker

* add test to check AV not called

* remove not needed docker ports

* linting

---------

Co-authored-by: Teemerte Teccola <teemerte.teccola@digital.trade.gov.uk>

.gitignore

@@ -86,7 +86,7 @@ ssl.crt
 node_modules
 dataworkspace/dataworkspace/static/*.css.map
 .local
-.localstack
+.localstack/mounted 
 test-results/
 cypress/screenshots/
 cypress/videos/

.localstack/Dockerfile

@@ -0,0 +1,3 @@
+FROM localstack/localstack:latest
+COPY ./init /etc/localstack/init/ready.d/
+RUN chmod +x /etc/localstack/init/ready.d/buckets.sh

.localstack/init/buckets.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -x
+awslocal s3 mb s3://notebooks.dataworkspace.local
+awslocal s3 mb s3://uploads.dataworkspace.local
+# Multipart file uploads from Your Files in the browser require ETag to be
+# exposed but by default it's not, so we add a CORS config to expose it
+awslocal s3api put-bucket-cors --bucket notebooks.dataworkspace.local --cors-configuration file:///etc/localstack/init/ready.d/s3-cors.json
+set +x
+
+echo "S3 Configured"

.localstack/init/s3-cors.json

@@ -0,0 +1,9 @@
+{
+  "CORSRules": [
+    {
+      "AllowedOrigins": ["http://dataworkspace.test:8000"],
+      "AllowedMethods": ["HEAD", "GET", "PUT", "POST", "DELETE"],
+      "ExposeHeaders": ["etag"]
+    }
+  ]
+}

dataworkspace/dataworkspace/apps/core/storage.py

@@ -97,9 +97,10 @@ def url(self, name):
 
 
 def malware_file_validator(file: FieldFile):
-    clamav_response = _upload_to_clamav(file)
+    if not settings.LOCAL:
+        clamav_response = _upload_to_clamav(file)
 
-    if clamav_response.malware:
-        msg = f"Virus found in {file.name}"
-        logger.error(msg)
-        raise ValidationError(msg, code="virus_found", params={"value": file})
+        if clamav_response.malware:
+            msg = f"Virus found in {file.name}"
+            logger.error(msg)
+            raise ValidationError(msg, code="virus_found", params={"value": file})

dataworkspace/dataworkspace/tests/core/test_storage.py

@@ -7,6 +7,7 @@
     S3FileStorage,
     ClamAVResponse,
     AntiVirusServiceErrorException,
+    malware_file_validator,
 )
 
 
@@ -49,3 +50,12 @@ def test_file_save_throws_exception_when_virus_found(
         fs.save("a-filename.txt", stream)
 
         mock_client().put_object.assert_not_called()
+
+
+@override_settings(LOCAL=True)
+@mock.patch("dataworkspace.apps.core.storage._upload_to_clamav")
+def test_malware_file_validator_not_called_on_local(
+    mock_upload_to_clamav,
+):
+    malware_file_validator("")
+    assert not mock_upload_to_clamav.called

docker-compose.yml

@@ -89,25 +89,22 @@ services:
 
   data-workspace-localstack:
     build:
-      context: localstack
+      context: .localstack
       dockerfile: Dockerfile
     networks:
       - data-infrastructure-shared-network
-    profiles:
-      - localstack
     ports:
-      - "127.0.0.1:53:53"
-      - "127.0.0.1:53:53/udp"
-      - "127.0.0.1:443:443"
       - '4563-4599:4563-4599'
     environment:
+      - DISABLE_CORS_CHECKS=1
+      - DISABLE_CUSTOM_CORS_S3=1
       - AWS_DEFAULT_REGION=eu-west-2
       - EDGE_PORT=4566
       - SERVICES=s3,sts,iam
-      - EXTRA_CORS_ALLOWED_ORIGINS=http://dataworkspace.test:8000
-      - DATA_DIR=/tmp/localstack/data
+      - EXTRA_CORS_ALLOWED_ORIGINS=http://dataworkspace.test:8000      
     volumes:
-      - "./.localstack:/tmp/localstack"
+      - "./.localstack/mounted:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
 
   # Superset profile services