feat: added antivirus integration (#171)

Makefile

@@ -38,7 +38,7 @@ integration-tests: ## Run go test with integration flags
 	 TEST_S3_ACCESS_KEY=$(shell make -s dev-s3-access-key) \
 	 TEST_S3_SECRET_KEY=$(shell make -s dev-s3-secret-key) \
 	 GIN_MODE=release \
-		richgo test -tags=integration $(GOTEST_OPTIONS) ./... # -run=UpdateFile/success
+		richgo test -tags=integration $(GOTEST_OPTIONS) ./... # -run=UploadFile/with_virus
 
 
 .PHONY: build
@@ -53,10 +53,31 @@ build-docker-image:  ## Build docker container for native architecture
 	./build/nix-docker-image.sh
 	docker tag hasura-storage:$(VERSION) hasura-storage:dev
 
+.PHONY: build-docker-image-clamav-dev
+build-docker-image-clamav-dev:  ## Build dev docker container for clamav
+	@echo $(VERSION) > VERSION
+	./build/nix-docker-image.sh clamavDockerImage
+	docker tag clamav:$(VERSION) clamav:dev
+
+.PHONY: build-docker-image-clamav
+build-docker-image-clamav:  ## Build docker container for clamav
+	@echo $(VERSION) > VERSION
+	./build/nix-docker-image.sh clamavDockerImage aarch64-linux
+	docker tag clamav:$(VERSION) nhost/clamav:$(VERSION)-aarch64
+	./build/nix-docker-image.sh clamavDockerImage x86_64-linux
+	docker tag clamav:$(VERSION) nhost/clamav:$(VERSION)-x86_64
+	docker push nhost/clamav:$(VERSION)-aarch64
+	docker push nhost/clamav:$(VERSION)-x86_64
+	docker manifest create \
+	  nhost/clamav:$(VERSION) \
+	    --amend nhost/clamav:$(VERSION)-aarch64 \
+	    --amend nhost/clamav:$(VERSION)-x86_64
+	docker manifest push nhost/clamav:$(VERSION)
+
 
 .PHONY: dev-env-up-short
 dev-env-up-short:  ## Starts development environment without hasura-storage
-	docker-compose -f ${DOCKER_DEV_ENV_PATH}/docker-compose.yaml up -d postgres graphql-engine minio
+	docker-compose -f ${DOCKER_DEV_ENV_PATH}/docker-compose.yaml up -d postgres graphql-engine minio clamd
 
 
 .PHONY: dev-env-up-hasura

README.md

@@ -50,6 +50,7 @@ The main features of the service are:
 - create presigned URLs to grant temporary access
 - caching information to integrate with caches and CDNs (cache headers, etag, conditional headers, etc)
 - perform basic image manipulation on the fly
+- integration with clamav antivirus
 
 ## OpenAPI
 

build/clamav/clamd.conf.tmpl

@@ -0,0 +1,12 @@
+Foreground yes
+
+DatabaseDirectory /clamav/db
+
+TCPSocket 3310
+
+MaxScanSize 1024M
+MaxFileSize 1024M
+StreamMaxLength 1024M
+
+MaxRecursion 16
+MaxFiles 10000

build/clamav/entrypoint.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -euo pipefail
+
+mkdir -p /clamav
+
+envsubst < /etc/clamav/freshclam.conf.tmpl > /etc/clamav/freshclam.conf
+envsubst < /etc/clamav/clamd.conf.tmpl > /etc/clamav/clamd.conf
+
+# we run freshclam first to download the database
+freshclam
+# we start the freshclam daemon
+freshclam -d &
+pid1=$!
+
+# we start the clamd daemon
+clamd &
+pid2=$!
+
+# Loop until either process finishes
+while true; do
+    if kill -0 $pid1 >/dev/null 2>&1; then
+        if kill -0 $pid2 >/dev/null 2>&1; then
+            sleep 5
+        else
+            kill $pid1
+            break
+        fi
+    else
+        kill $pid2
+        break
+    fi
+done
+
+exit 1

build/clamav/freshclam.conf.tmpl

@@ -0,0 +1,5 @@
+DatabaseDirectory /clamav/db
+Foreground yes
+DatabaseOwner root
+DatabaseMirror ${DATABASE_MIRROR:-database.clamav.net}
+NotifyClamd yes

build/dev/docker/docker-compose.yaml

@@ -62,4 +62,12 @@ services:
       S3_ROOT_FOLDER: "f215cf48-7458-4596-9aa5-2159fc6a3caf"
       POSTGRES_MIGRATIONS: 1
       POSTGRES_MIGRATIONS_SOURCE: ${HASURA_GRAPHQL_DATABASE_URL:-postgres://postgres:hejsan@postgres:5432/postgres?sslmode=disable}
+      CLAMAV_SERVER: tcp://clamd:3310
     command: serve
+
+  clamd:
+    container_name: hasura-storage-clamd
+    image: nhost/clamav:0.1.0
+    restart: unless-stopped
+    ports:
+      - '3310:3310'

build/nix-docker-image.sh

@@ -2,22 +2,27 @@
 
 which nix > /dev/null
 
+IMAGE=${1:-"dockerImage"}
+SYSTEM=${2:-""}
+
 if [[ $NIX_BUILD_NATIVE -eq 1 ]]; then
-    case $(uname -m) in
-    "arm64")
-        SYSTEM="aarch64-linux"
-        ;;
-    *)
-        SYSTEM="x86_64-linux"
-        ;;
-    esac
+    if [[ -z $SYSTEM ]]; then
+        case $(uname -m) in
+        "arm64")
+            SYSTEM="aarch64-linux"
+            ;;
+        *)
+            SYSTEM="x86_64-linux"
+            ;;
+        esac
+    fi
 
-    nix build .\#packages.${SYSTEM}.dockerImage --print-build-logs && docker load < result
+    nix build .\#packages.${SYSTEM}.$IMAGE --print-build-logs && docker load < result
     exit $?
 fi
 
 if [[ ( $? -eq 0 ) && ( `uname` == "Linux" ) ]]; then
-    nix build .\#dockerImage --print-build-logs && docker load < result
+    nix build .\#$IMAGE --print-build-logs && docker load < result
     exit $?
 fi
 
@@ -28,4 +33,4 @@ docker run --rm -it \
     -w /build \
     --entrypoint sh \
     dbarroso/nix:2.6.0 \
-        -c "nix build .\\#dockerImage --print-build-logs && docker load < result"
+        -c "nix build .\\#packages.$IMAGE --print-build-logs && docker load < result"

clamd/clamd.go

@@ -0,0 +1,87 @@
+package clamd
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"time"
+)
+
+const chunkSize = 1024
+
+type Client struct {
+	addr string
+}
+
+func NewClient(addr string) (*Client, error) {
+	url, err := url.Parse(addr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse addr: %w", err)
+	}
+
+	if url.Scheme != "tcp" {
+		return nil, fmt.Errorf("invalid scheme: %s", url.Scheme) //nolint:goerr113
+	}
+
+	return &Client{url.Host}, nil
+}
+
+func (c *Client) Dial() (net.Conn, error) {
+	conn, err := net.Dial("tcp", c.addr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to dial: %w", err)
+	}
+
+	if err := conn.SetDeadline(time.Now().Add(1 * time.Minute)); err != nil {
+		return nil, fmt.Errorf("failed to set deadline: %w", err)
+	}
+
+	return conn, nil
+}
+
+func sendCommand(conn net.Conn, command string) error {
+	if _, err := conn.Write(
+		[]byte(fmt.Sprintf("n%s\n", command)),
+	); err != nil {
+		return fmt.Errorf("failed to write command: %w", err)
+	}
+	return nil
+}
+
+func readResponse(conn net.Conn) ([]byte, error) {
+	buf := make([]byte, 1024) //nolint:gomnd
+	n, err := conn.Read(buf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response: %w", err)
+	}
+	return buf[:n], nil
+}
+
+func sendChunk(conn net.Conn, data []byte) error {
+	var buf [4]byte
+	lenData := len(data)
+	buf[0] = byte(lenData >> 24) //nolint:gomnd
+	buf[1] = byte(lenData >> 16) //nolint:gomnd
+	buf[2] = byte(lenData >> 8)  //nolint:gomnd
+	buf[3] = byte(lenData >> 0)
+
+	a := buf
+
+	b := make([]byte, len(a))
+	copy(b, a[:])
+
+	if _, err := conn.Write(b); err != nil {
+		return fmt.Errorf("failed to write chunk size: %w", err)
+	}
+
+	if _, err := conn.Write(data); err != nil {
+		return fmt.Errorf("failed to write chunk: %w", err)
+	}
+
+	return nil
+}
+
+func sendEOF(conn net.Conn) error {
+	_, err := conn.Write([]byte{0, 0, 0, 0})
+	return err //nolint:wrapcheck
+}

clamd/errors.go

@@ -0,0 +1,9 @@
+package clamd
+
+type VirusFoundError struct {
+	Name string
+}
+
+func (e *VirusFoundError) Error() string {
+	return "virus found: " + e.Name
+}

clamd/instream.go

@@ -0,0 +1,55 @@
+package clamd
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+func (c *Client) InStream(r io.ReaderAt) error { //nolint: cyclop
+	conn, err := c.Dial()
+	if err != nil {
+		return fmt.Errorf("failed to dial: %w", err)
+	}
+	defer conn.Close()
+
+	if err := sendCommand(conn, "INSTREAM"); err != nil {
+		return fmt.Errorf("failed to send INSTREAM command: %w", err)
+	}
+
+	var iter int64
+	for {
+		buf := make([]byte, chunkSize)
+
+		nr, err := r.ReadAt(buf, iter*chunkSize)
+		iter++
+
+		if nr > 0 {
+			if err := sendChunk(conn, buf[0:nr]); err != nil {
+				return fmt.Errorf("failed to send chunk: %w", err)
+			}
+		}
+
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("failed to read chunk: %w", err)
+		}
+	}
+
+	if err := sendEOF(conn); err != nil {
+		return fmt.Errorf("failed to send EOF: %w", err)
+	}
+
+	response, err := readResponse(conn)
+	if err != nil {
+		return fmt.Errorf("failed to read response: %w", err)
+	}
+
+	if string(response) == "stream: OK\n" {
+		return nil
+	}
+
+	return &VirusFoundError{string(response[8 : len(response)-7])}
+}

clamd/instream_test.go

@@ -0,0 +1,54 @@
+package clamd_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/nhost/hasura-storage/clamd"
+)
+
+func TestClamdInstream(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name          string
+		filepath      string
+		expectedError error
+	}{
+		{
+			name:     "clean",
+			filepath: "clamd.go",
+		},
+		{
+			name:     "eicarcom2.zip",
+			filepath: "testdata/eicarcom2.zip",
+			expectedError: &clamd.VirusFoundError{
+				Name: "Win.Test.EICAR_HDB-1",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			client, err := clamd.NewClient("tcp://localhost:3310")
+			if err != nil {
+				t.Fatalf("failed to dial: %v", err)
+			}
+
+			f, err := os.Open(tc.filepath)
+			if err != nil {
+				t.Fatalf("failed to open file: %v", err)
+			}
+			defer f.Close()
+
+			err = client.InStream(f)
+			if diff := cmp.Diff(tc.expectedError, err); diff != "" {
+				t.Errorf("unexpected error (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

clamd/ping.go

@@ -0,0 +1,26 @@
+package clamd
+
+import "fmt"
+
+func (c *Client) Ping() error {
+	conn, err := c.Dial()
+	if err != nil {
+		return fmt.Errorf("failed to dial: %w", err)
+	}
+	defer conn.Close()
+
+	if err := sendCommand(conn, "PING"); err != nil {
+		return fmt.Errorf("failed to send PING command: %w", err)
+	}
+
+	response, err := readResponse(conn)
+	if err != nil {
+		return fmt.Errorf("failed to read response: %w", err)
+	}
+
+	if string(response) != "PONG\n" {
+		return fmt.Errorf("unknown response: %s", string(response)) //nolint:goerr113
+	}
+
+	return nil
+}