Run av scanning from ClamD server

services/app-web/src/s3/s3Amplify.ts

@@ -99,16 +99,16 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT {
         },
         /*  
             Poll for scanning completion
-            - We start polling after 20s, which is the estimated time it takes scanning to start to resolve.
-            - In total, each file could be up to 40 sec in a loading state (20s wait for scanning + 8s of retries + extra time for uploading and scanning api requests to resolve)
+            - We start polling after 3s, which is the estimated time it takes scanning to start to resolve.
+            - We then retry with an exponential backoff for up to 15s. Most scans take < 1s, plus some additional time for tagging and response, so by 15s a file should be tagged.
             - While the file is scanning, returns 403. When scanning is complete, the resource returns 200
         */
         scanFile: async (
             filename: string,
             bucket: BucketShortName
         ): Promise<void | S3Error> => {
             try {
-                await waitFor(20000)
+                await waitFor(3000)
                 try {
                     await retryWithBackoff(async () => {
                         await Storage.get(filename, {
@@ -213,7 +213,7 @@ const waitFor = (delay = 1000) =>
 const retryWithBackoff = async (
     fn: () => Promise<void | S3Error>,
     retryCount = 0,
-    maxRetries = 6,
+    maxRetries = 4,
     err: null | S3Error = null
 ): Promise<void | S3Error> => {
     if (retryCount > maxRetries) {

services/uploads/README.md

@@ -16,6 +16,32 @@ In addition to live scanning all uploaded files, we also have a pair of lambdas
 
 The avAuditUploads lambda pulls a list of every file in the uploads bucket and then invokes avAuditFiles for each of them in chunks of 20 or so. For any files that are found to be INFECTED, it then grabs the current s3 tags for them and verifies that they are tagged accordingly. If not, it will re-tag the file to be INFECTED, preventing further download.
 
+## ClamAV Daemon
+
+We have an ec2 instance that is created in dev, val, and prod that is configured with an always on ClamAV instance that accepts incoming virus scan requests on port 3310. The motivation here is that we are working towards having our av scanning lambdas use the always on ClamAV daemon rather than rely on just the lambda, as there is a high startup cost for ClamAV of around 29 seconds in our testing. This means that all virus scans take at least 29 seconds for the user. By using an always on instance we can reduce that to closer to the actual time of the virus scan (usually < 1s).
+
+The server is restricted to only have access from connections in the default security group as well as anything else that is placed in the ClamAV security group. This allows for our AV scanning lambds to call out to the instance while restricting all other traffic. However, all of our engineers have ssh pub keys on the instance in case they need access to the machine via ssh for any reason.
+
+### Accessing the VM
+
+Similar to the [Postgres jumpbox](../postgres/README.md), we use the `authorized_keys` file to give access to this VM and you'll need to add your IP to the VM's security group:
+
+1. Determine your public facing IP address. An easy way to do this is to `curl https://ifconfig.me/`
+2. Locate the EC2 instance in the AWS console. Click and go into Security > Security groups.
+3. There should be two security groups attached to the instance, the default and the ClamAV one. Select the ClamAV security group.
+4. On the `Inbound rules` tab select `Edit inbound rules`
+5. Add a rule for `ssh` with the `source` set to your local IP address with `/32` appended to it (e.g. `1.2.3.4/32`)
+6. Save the rule
+
+#### SSH to the instances
+
+You should now be able to ssh to the jump box.
+
+1. Locate the Public IPv4 address of the instance. This can be found by clicking into the VM on the `Instances` section of the EC2 console.
+2. ssh ubuntu@public-ip
+
+You should be using public key auth to ssh. If you need to point to your private key, use `ssh -i ~/.ssh/${yourkeyfile} ubuntu@public-ip`
+
 ## Significant dependencies
 
 -   serverless-s3-upload

services/uploads/serverless.yml

@@ -52,20 +52,19 @@ provider:
 custom:
   region: ${aws:region}
   reactAppOtelCollectorUrl: ${env:REACT_APP_OTEL_COLLECTOR_URL, ssm:/configuration/react_app_otel_collector_url}
+  authorizedKeys: ${file(../postgres/scripts/authorized_keys)}
   webpack:
     webpackConfig: webpack.config.js
     packager: yarn
     packagerOptions:
       lockFile: ../../yarn.lock
-  scripts:
-    hooks:
-      # This script is run locally when running 'serverless deploy'
-      package:initialize: |
-        set -e
-        curl -L --output lambda_layer.zip https://github.com/CMSgov/lambda-clamav-layer/releases/download/0.7/lambda_layer.zip
-      deploy:finalize: |
-        rm lambda_layer.zip
-        serverless invoke --stage ${sls:stage} --function avDownloadDefinitions -t Event
+  vpcId: ${ssm:/configuration/${sls:stage}/vpc/id, ssm:/configuration/default/vpc/id}
+  sgId: ${ssm:/configuration/${sls:stage}/vpc/sg/id, ssm:/configuration/default/vpc/sg/id}
+  privateSubnets:
+    - ${ssm:/configuration/${sls:stage}/vpc/subnets/private/a/id, ssm:/configuration/default/vpc/subnets/private/a/id}
+    - ${ssm:/configuration/${sls:stage}/vpc/subnets/private/b/id, ssm:/configuration/default/vpc/subnets/private/b/id}
+    - ${ssm:/configuration/${sls:stage}/vpc/subnets/private/c/id, ssm:/configuration/default/vpc/subnets/private/c/id}
+  publicSubnetA: ${ssm:/configuration/${sls:stage}/vpc/subnets/public/a/id, ssm:/configuration/default/vpc/subnets/public/a/id}
   serverless-offline-ssm:
     stages:
       - local
@@ -96,8 +95,7 @@ custom:
 
 layers:
   clamAv:
-    package:
-      artifact: lambda_layer.zip
+    path: lambda-layers-clamav
 
 functions:
   avScan:
@@ -109,6 +107,9 @@ functions:
     layers:
       - !Ref ClamAvLambdaLayer
       - arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:1
+    vpc:
+      securityGroupIds: ${self:custom.sgId}
+      subnetIds: ${self:custom.privateSubnets}
     environment:
       stage: ${sls:stage}
       CLAMAV_BUCKET_NAME: !Ref ClamDefsBucket
@@ -171,6 +172,13 @@ functions:
       REACT_APP_OTEL_COLLECTOR_URL: ${self:custom.reactAppOtelCollectorUrl}
 
 resources:
+  Conditions:
+    IsDevValProd: !Or
+      - !Equals ['${sls:stage}', 'main']
+      - !Equals ['${sls:stage}', 'val']
+      - !Equals ['${sls:stage}', 'prod']
+      - !Equals ['${sls:stage}', 'mtscanfromlambda']
+
   Resources:
     DocumentUploadsBucket:
       Type: AWS::S3::Bucket
@@ -364,6 +372,132 @@ resources:
                 - !Sub ${QAUploadsBucket.Arn}/*
               Sid: DenyUnencryptedConnections
 
+    ClamAVSecurityGroup:
+      Type: AWS::EC2::SecurityGroup
+      Properties:
+        GroupDescription: Security group for ClamAV daemon
+        VpcId: ${self:custom.vpcId}
+        SecurityGroupIngress:
+          - IpProtocol: tcp
+            FromPort: 3310
+            ToPort: 3310
+            SourceSecurityGroupId: ${self:custom.sgId}
+
+    ClamAVInstanceProfile:
+      Type: AWS::IAM::InstanceProfile
+      Properties:
+        Path: '/delegatedadmin/developer/'
+        Roles:
+          - !Ref ClamAVInstanceRole
+
+    ClamAVInstanceRole:
+      Type: AWS::IAM::Role
+      Properties:
+        Path: '/delegatedadmin/developer/'
+        PermissionsBoundary: !Sub 'arn:aws:iam::${AWS::AccountId}:policy/cms-cloud-admin/ct-ado-poweruser-permissions-boundary-policy'
+        RoleName: !Sub 'clamavdVm-${sls:stage}-ServiceRole'
+        AssumeRolePolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+            - Effect: Allow
+              Principal:
+                Service: ec2.amazonaws.com
+              Action: sts:AssumeRole
+        Policies:
+          - PolicyName: ClamAVInstancePolicy
+            PolicyDocument:
+              Version: '2012-10-17'
+              Statement:
+                - Effect: Allow
+                  Action:
+                    - logs:CreateLogGroup
+                    - logs:CreateLogStream
+                    - logs:PutLogEvents
+                  Resource: '*'
+
+    ClamAVInstance:
+      Type: AWS::EC2::Instance
+      Condition: IsDevValProd
+      Properties:
+        InstanceType: t3.medium
+        ImageId: ami-0c7217cdde317cfec # Ubuntu 22.04 LTS
+        IamInstanceProfile: !Ref ClamAVInstanceProfile
+        NetworkInterfaces:
+          - AssociatePublicIpAddress: true
+            DeviceIndex: '0'
+            GroupSet:
+              - !Ref ClamAVSecurityGroup
+            SubnetId: !Sub ${self:custom.publicSubnetA}
+        Tags:
+          - Key: Name
+            Value: clamavd-${sls:stage}
+          - Key: mcr-vmuse
+            Value: clamavd
+        UserData:
+          Fn::Base64: !Sub |
+            #!/bin/bash
+            apt-get update
+            apt-get install -y clamav clamav-daemon
+
+            echo '${self:custom.authorizedKeys}' > /home/ubuntu/.ssh/authorized_keys
+            chown ubuntu:ubuntu /home/ubuntu/.ssh/authorized_keys
+            chmod 600 /home/ubuntu/.ssh/authorized_keys
+
+            # Write to the clamd.conf
+            echo "TCPSocket 3310" >> /etc/clamav/clamd.conf
+            echo "TCPAddr 0.0.0.0" >> /etc/clamav/clamd.conf 
+
+            # Create a systemd service override to delay the start
+            cat <<EOF > /etc/systemd/system/clamav-daemon.service.d/override.conf
+            [Unit]
+            After=network.target
+            EOF
+
+            # Create a systemd service override to delay the start and set restart limits
+            cat <<EOF > /etc/systemd/system/clamav-daemon.service.d/override.conf
+            [Unit]
+            After=network.target
+            StartLimitIntervalSec=1h
+            StartLimitBurst=5
+            EOF
+
+            # Fix the systemctl setting
+            sed -i 's/^StandardOutput=syslog/StandardOutput=journal/' /lib/systemd/system/clamav-daemon.service
+
+            # Reload systemd to apply the changes
+            systemctl daemon-reload
+
+            # Start clamd and get defs
+            systemctl enable clamav-daemon
+            systemctl enable clamav-freshclam
+            systemctl start clamav-daemon
+            systemctl start clamav-freshclam
+
+            # Confirm we're up
+            systemctl status clamav-daemon
+            systemctl status clamav-freshclam
+
+    MCRInternalZone:
+      Type: AWS::Route53::HostedZone
+      Condition: IsDevValProd
+      Properties:
+        Name: mc-review.local
+        VPCs:
+          - VPCId: ${self:custom.vpcId}
+            VPCRegion: !Ref AWS::Region
+
+    ClamAVRecordSet:
+      Type: AWS::Route53::RecordSet
+      Condition: IsDevValProd
+      DependsOn: ClamAVInstance
+      Properties:
+        HostedZoneId: !Ref MCRInternalZone
+        Name: clamav.mc-review.local
+        Type: A
+        ResourceRecords:
+          - !GetAtt ClamAVInstance.PrivateIp
+        TTL: '300'
+
   Outputs:
     DocumentUploadsBucketName:
       Value: !Ref DocumentUploadsBucket

services/uploads/src/avLayer/build/build.sh

@@ -51,6 +51,7 @@ cp /tmp/build/usr/local/bin/clamscan /tmp/build/usr/local/bin/clamdscan /tmp/bui
 cp -R /tmp/build/usr/lib64/* lib/.
 cp -R /tmp/build/usr/local/lib64/* lib/.
 cp freshclam.conf bin/freshclam.conf
+cp clamd.conf bin/clamd.conf
 
 zip -r9 lambda_layer.zip bin
 zip -r9 lambda_layer.zip lib

services/uploads/src/avLayer/build/clamd.conf

@@ -0,0 +1,12 @@
+# hostname and port of the remote ClamAV daemon
+TCPAddr clamav.mc-review.local
+TCPSocket  3310
+
+# Enable verbose logging
+LogVerbose yes
+
+# Path to the log file 
+LogFile /var/log/clamd.log
+
+# Set the maximum number of concurrent threads for scanning
+MaxThreads  10

services/uploads/src/deps/clamAV/clamAV.ts

@@ -21,6 +21,9 @@ interface ClamAVConfig {
     pathToFreshclam: string
     pathToConfig: string
     pathToDefintions: string
+
+    pathToClamdScan: string
+    pathToClamdConfig: string
 }
 
 function NewClamAV(config: Partial<ClamAVConfig>, s3Client: S3UploadsClient) {
@@ -36,6 +39,9 @@ function NewClamAV(config: Partial<ClamAVConfig>, s3Client: S3UploadsClient) {
         pathToFreshclam: config.pathToFreshclam || '/opt/bin/freshclam',
         pathToConfig: config.pathToConfig || '/opt/bin/freshclam.conf',
         pathToDefintions: config.pathToDefintions || '/tmp',
+
+        pathToClamdScan: config.pathToClamdScan || '/opt/bin/clamdscan',
+        pathToClamdConfig: config.pathToClamdConfig || '/opt/bin/clamd.conf',
     }
 
     return {
@@ -177,13 +183,24 @@ function scanForInfectedFiles(
     try {
         console.info('Executing clamav')
 
+        // use clamdscan to connect to our clamavd server
+        const avResult = spawnSync(config.pathToClamdScan, [
+            '--stdout',
+            '-v',
+            `--config-file=${config.pathToClamdConfig}`,
+            '--stream',
+            pathToScan,
+        ])
+
+        /*
         const avResult = spawnSync(config.pathToClamav, [
             '--stdout',
             '-v',
             '-d',
             config.pathToDefintions,
             pathToScan,
         ])
+        */
 
         console.info('stderror', avResult.stderr && avResult.stderr.toString())
         console.info('stdout', avResult.stdout && avResult.stdout.toString())

services/uploads/src/lib/scanFiles.ts

@@ -12,14 +12,6 @@ export async function scanFiles(
     bucket: string,
     scanDir: string
 ): Promise<string[] | Error> {
-    // fetch definition files
-    console.info('Download AV Definitions')
-    const defsRes = await clamAV.downloadAVDefinitions()
-    if (defsRes) {
-        console.error('failed to fetch definitions')
-        return defsRes
-    }
-
     // clamScan wants files to be top level in the scanned directory, so we map each key to a UUID
     const filemap: { [filename: string]: string } = {}
     for (const key of keys) {