From 1e814941916bb70e65bb31b4e4d8ddeff354d31b Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 1 Feb 2024 16:36:01 -0500
Subject: [PATCH 01/38] try to add a new vm for clamavd

---
 services/uploads/serverless.yml | 71 +++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 0a59aee6f2..09888ad54d 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -66,6 +66,13 @@ custom:
       deploy:finalize: |
         rm lambda_layer.zip
         serverless invoke --stage ${sls:stage} --function avDownloadDefinitions -t Event
+  vpcId: ${ssm:/configuration/${sls:stage}/vpc/id, ssm:/configuration/default/vpc/id}
+  sgId: ${ssm:/configuration/${sls:stage}/vpc/sg/id, ssm:/configuration/default/vpc/sg/id}
+  privateSubnets:
+    - ${ssm:/configuration/${sls:stage}/vpc/subnets/private/a/id, ssm:/configuration/default/vpc/subnets/private/a/id}
+    - ${ssm:/configuration/${sls:stage}/vpc/subnets/private/b/id, ssm:/configuration/default/vpc/subnets/private/b/id}
+    - ${ssm:/configuration/${sls:stage}/vpc/subnets/private/c/id, ssm:/configuration/default/vpc/subnets/private/c/id}
+  publicSubnetA: ${ssm:/configuration/${sls:stage}/vpc/subnets/public/a/id, ssm:/configuration/default/vpc/subnets/public/a/id}
   serverless-offline-ssm:
     stages:
       - local
@@ -109,6 +116,9 @@ functions:
     layers:
       - !Ref ClamAvLambdaLayer
       - arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:1
+    vpc:
+      securityGroupIds: ${self:custom.sgId}
+      subnetIds: ${self:custom.privateSubnets}
     environment:
       stage: ${sls:stage}
       CLAMAV_BUCKET_NAME: !Ref ClamDefsBucket
@@ -364,6 +374,67 @@ resources:
                 - !Sub ${QAUploadsBucket.Arn}/*
               Sid: DenyUnencryptedConnections
 
+    ClamAVSecurityGroup:
+      Type: AWS::EC2::SecurityGroup
+      Properties:
+        GroupDescription: Security group for ClamAV daemon
+        SecurityGroupIngress:
+          - IpProtocol: tcp
+            FromPort: 3310
+            ToPort: 3310
+            SourceSecurityGroupId: ${self:custom.sgId}
+
+    ClamAVInstanceProfile:
+      Type: AWS::IAM::InstanceProfile
+      Properties:
+        Path: /
+        Roles:
+          - !Ref ClamAVInstanceRole
+
+    ClamAVInstanceRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+            - Effect: Allow
+              Principal:
+                Service: lambda.amazonaws.com
+              Action: sts:AssumeRole
+        Policies:
+          - PolicyName: ClamAVInstancePolicy
+            PolicyDocument:
+              Version: '2012-10-17'
+              Statement:
+                - Effect: Allow
+                  Action:
+                    - logs:CreateLogGroup
+                    - logs:CreateLogStream
+                    - logs:PutLogEvents
+                  Resource: '*'
+
+    ClamAVInstance:
+      Type: AWS::EC2::Instance
+      Properties:
+        InstanceType: t2.micro
+        ImageId: ami-05bfc1ab11bfbf484
+        KeyName: my-key-pair # Replace with your key pair name
+        SecurityGroups:
+          - !Ref ClamAVSecurityGroup
+        IamInstanceProfile: !Ref ClamAVInstanceProfile
+        Tags:
+          - Key: mcr-vmuse
+            Value: clamavd
+        UserData:
+          Fn::Base64: !Sub |
+            #!/bin/bash
+            sudo apt-get update
+            sudo apt-get install -y clamav clamav-daemon
+            sudo systemctl enable clamav-freshclam
+            sudo systemctl start clamav-freshclam
+            sudo systemctl enable clamav-daemon
+            sudo systemctl start clamav-daemon
+
   Outputs:
     DocumentUploadsBucketName:
       Value: !Ref DocumentUploadsBucket

From dc8058cc2c8e8b098eb66dec39edcb0511a5f3c2 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 1 Feb 2024 16:56:23 -0500
Subject: [PATCH 02/38] add vpcid to sg

---
 services/uploads/serverless.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 09888ad54d..34eb79b234 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -378,6 +378,7 @@ resources:
       Type: AWS::EC2::SecurityGroup
       Properties:
         GroupDescription: Security group for ClamAV daemon
+        VpcId: ${self:custom.vpcId}
         SecurityGroupIngress:
           - IpProtocol: tcp
             FromPort: 3310
@@ -418,7 +419,6 @@ resources:
       Properties:
         InstanceType: t2.micro
         ImageId: ami-05bfc1ab11bfbf484
-        KeyName: my-key-pair # Replace with your key pair name
         SecurityGroups:
           - !Ref ClamAVSecurityGroup
         IamInstanceProfile: !Ref ClamAVInstanceProfile

From 743bb4591cae5a5a25479458bac5f3073a5c26a4 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 1 Feb 2024 17:15:49 -0500
Subject: [PATCH 03/38] add perm boundary

---
 services/uploads/serverless.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 34eb79b234..ca82f675a8 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -395,6 +395,9 @@ resources:
     ClamAVInstanceRole:
       Type: AWS::IAM::Role
       Properties:
+        Path: '/delegatedadmin/developer/'
+        PermissionsBoundary: !Sub 'arn:aws:iam::${AWS::AccountId}:policy/cms-cloud-admin/ct-ado-poweruser-permissions-boundary-policy'
+        RoleName: !Sub 'clamavdVm-${sls:stage}-ServiceRole'
         AssumeRolePolicyDocument:
           Version: '2012-10-17'
           Statement:

From aa3669c93a56fb07b445fde5af731b1c1056dff9 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 2 Feb 2024 13:14:24 -0500
Subject: [PATCH 04/38] fix path for iam

---
 services/uploads/serverless.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index ca82f675a8..e1b2924ff4 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -388,7 +388,7 @@ resources:
     ClamAVInstanceProfile:
       Type: AWS::IAM::InstanceProfile
       Properties:
-        Path: /
+        Path: '/delegatedadmin/developer/'
         Roles:
           - !Ref ClamAVInstanceRole
 
@@ -403,7 +403,7 @@ resources:
           Statement:
             - Effect: Allow
               Principal:
-                Service: lambda.amazonaws.com
+                Service: ec2.amazonaws.com
               Action: sts:AssumeRole
         Policies:
           - PolicyName: ClamAVInstancePolicy

From 9ec6aca297882df60deaa5ae0975c107e92d1aa7 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 2 Feb 2024 14:25:16 -0500
Subject: [PATCH 05/38] add a network interface

---
 services/uploads/serverless.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index e1b2924ff4..6ca78a2873 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -425,6 +425,13 @@ resources:
         SecurityGroups:
           - !Ref ClamAVSecurityGroup
         IamInstanceProfile: !Ref ClamAVInstanceProfile
+        NetworkInterfaces:
+          - AssociatePublicIpAddress: true
+            DeviceIndex: '0'
+            GroupSet:
+              - !Ref ClamAvSecurityGroup
+              - ${self:custom.sgId}
+            SubnetId: !Sub ${self:custom.publicSubnetA}
         Tags:
           - Key: mcr-vmuse
             Value: clamavd

From 9459329e1a1b098e3cb6b2eb022bc50c26d89fba Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 2 Feb 2024 14:45:43 -0500
Subject: [PATCH 06/38] fixing typo

---
 services/uploads/serverless.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 6ca78a2873..7fa786f7bd 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -429,7 +429,7 @@ resources:
           - AssociatePublicIpAddress: true
             DeviceIndex: '0'
             GroupSet:
-              - !Ref ClamAvSecurityGroup
+              - !Ref ClamAVSecurityGroup
               - ${self:custom.sgId}
             SubnetId: !Sub ${self:custom.publicSubnetA}
         Tags:

From 5bb18de4b53eb213e4945eb415ddbef07c301505 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 5 Feb 2024 15:59:17 -0500
Subject: [PATCH 07/38] add name to vm

---
 services/uploads/serverless.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 7fa786f7bd..ff6ecd1704 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -422,8 +422,6 @@ resources:
       Properties:
         InstanceType: t2.micro
         ImageId: ami-05bfc1ab11bfbf484
-        SecurityGroups:
-          - !Ref ClamAVSecurityGroup
         IamInstanceProfile: !Ref ClamAVInstanceProfile
         NetworkInterfaces:
           - AssociatePublicIpAddress: true
@@ -433,6 +431,8 @@ resources:
               - ${self:custom.sgId}
             SubnetId: !Sub ${self:custom.publicSubnetA}
         Tags:
+          - Key: Name
+            Value: clamavd-${sls:stage}
           - Key: mcr-vmuse
             Value: clamavd
         UserData:

From 9cf4a7240cab24d1c7fa3e76084557d781f43ad2 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 5 Feb 2024 16:02:52 -0500
Subject: [PATCH 08/38] only make this server in dev val prod

---
 services/uploads/serverless.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index ff6ecd1704..a121f35dad 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -181,6 +181,13 @@ functions:
       REACT_APP_OTEL_COLLECTOR_URL: ${self:custom.reactAppOtelCollectorUrl}
 
 resources:
+  Conditions:
+    IsDevValProd: !Or
+      - !Equals ['${sls:stage}', 'main']
+      - !Equals ['${sls:stage}', 'val']
+      - !Equals ['${sls:stage}', 'prod']
+      - !Equals ['${sls:stage}', 'mtbuildclamavserver']
+
   Resources:
     DocumentUploadsBucket:
       Type: AWS::S3::Bucket
@@ -419,6 +426,7 @@ resources:
 
     ClamAVInstance:
       Type: AWS::EC2::Instance
+      Condition: IsDevValProd
       Properties:
         InstanceType: t2.micro
         ImageId: ami-05bfc1ab11bfbf484

From 86fc9fc41d3cae167712cec459baff8da3c6fa98 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 5 Feb 2024 16:19:54 -0500
Subject: [PATCH 09/38] bump and test

---
 services/uploads/serverless.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index a121f35dad..039972f373 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -428,9 +428,10 @@ resources:
       Type: AWS::EC2::Instance
       Condition: IsDevValProd
       Properties:
-        InstanceType: t2.micro
+        InstanceType: t3.medium
         ImageId: ami-05bfc1ab11bfbf484
         IamInstanceProfile: !Ref ClamAVInstanceProfile
+        KeyName: mojo # for testing
         NetworkInterfaces:
           - AssociatePublicIpAddress: true
             DeviceIndex: '0'

From 2d61fc21f69ed6a6d7068c3696891315c329f536 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 5 Feb 2024 16:52:46 -0500
Subject: [PATCH 10/38] bump ami

---
 services/uploads/serverless.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 039972f373..f31a3d9c2b 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -429,7 +429,7 @@ resources:
       Condition: IsDevValProd
       Properties:
         InstanceType: t3.medium
-        ImageId: ami-05bfc1ab11bfbf484
+        ImageId: ami-0c7217cdde317cfec # Ubuntu 22.04 LTS
         IamInstanceProfile: !Ref ClamAVInstanceProfile
         KeyName: mojo # for testing
         NetworkInterfaces:

From a818d41a3d11e221fe9c260041d36f3424fa5ad4 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 5 Feb 2024 17:18:31 -0500
Subject: [PATCH 11/38] user data clamav latest LTS

---
 services/uploads/serverless.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index f31a3d9c2b..99bf1cdc63 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -447,8 +447,20 @@ resources:
         UserData:
           Fn::Base64: !Sub |
             #!/bin/bash
-            sudo apt-get update
-            sudo apt-get install -y clamav clamav-daemon
+            apt-get update
+            apt-get install -y clamav clamav-daemon
+
+            # Install clamav 1.0.4
+            wget https://www.clamav.net/downloads/production/clamav-1.0.4.linux.x86_64.deb
+            dpkg -i clamav-1.0.4.linux.x86_64.deb
+            apt-get install -f
+
+            # Write the freshclam.conf
+            echo "DatabaseMirror database.clamav.net" > /usr/local/etc/freshclam.conf
+            echo "CompressLocalDatabase yes" >> /usr/local/etc/freshclam.conf
+            chmod  644 /usr/local/etc/freshclam.conf
+            chown clamav:clamav /usr/local/etc/freshclam.conf
+
             sudo systemctl enable clamav-freshclam
             sudo systemctl start clamav-freshclam
             sudo systemctl enable clamav-daemon

From 6cee3239dea29c5ad2a1293efd1a58a0add3cfcb Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Wed, 7 Feb 2024 17:11:35 -0500
Subject: [PATCH 12/38] fix up clamav configs

---
 services/uploads/serverless.yml | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 99bf1cdc63..bcd18a9b78 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -455,16 +455,15 @@ resources:
             dpkg -i clamav-1.0.4.linux.x86_64.deb
             apt-get install -f
 
-            # Write the freshclam.conf
-            echo "DatabaseMirror database.clamav.net" > /usr/local/etc/freshclam.conf
-            echo "CompressLocalDatabase yes" >> /usr/local/etc/freshclam.conf
-            chmod  644 /usr/local/etc/freshclam.conf
-            chown clamav:clamav /usr/local/etc/freshclam.conf
+            # Write to the clamd.conf
+            echo "TCPSocket 3310" >> /etc/clamav/clamd.conf
+            echo "TCPAddr 0.0.0.0" >> /etc/clamav/clamd.conf 
 
-            sudo systemctl enable clamav-freshclam
-            sudo systemctl start clamav-freshclam
-            sudo systemctl enable clamav-daemon
-            sudo systemctl start clamav-daemon
+            # Start clamd and get defs
+            systemctl enable clamav-freshclam
+            systemctl start clamav-freshclam
+            systemctl enable clamav-daemon
+            systemctl start clamav-daemon
 
   Outputs:
     DocumentUploadsBucketName:

From 399e8a65009f5715665ac8916b9b41818d79731d Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 8 Feb 2024 12:37:58 -0500
Subject: [PATCH 13/38] add all keys to user data

---
 services/uploads/serverless.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index bcd18a9b78..9de2d1a42c 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -52,6 +52,7 @@ provider:
 custom:
   region: ${aws:region}
   reactAppOtelCollectorUrl: ${env:REACT_APP_OTEL_COLLECTOR_URL, ssm:/configuration/react_app_otel_collector_url}
+  authorizedKeys: ${file(../postgres/scripts/authorized_keys)}
   webpack:
     webpackConfig: webpack.config.js
     packager: yarn
@@ -450,6 +451,10 @@ resources:
             apt-get update
             apt-get install -y clamav clamav-daemon
 
+            echo '${self:custom.authorizedKeys}' > /home/ubuntu/.ssh/authorized_keys
+            chown ubuntu:ubuntu /home/ubuntu/.ssh/authorized_keys
+            chmod 600 /home/ubuntu/.ssh/authorized_keys
+
             # Install clamav 1.0.4
             wget https://www.clamav.net/downloads/production/clamav-1.0.4.linux.x86_64.deb
             dpkg -i clamav-1.0.4.linux.x86_64.deb

From 7beae944861ad214df4aeb44d105fc9b58566010 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 8 Feb 2024 13:58:24 -0500
Subject: [PATCH 14/38] Readme

---
 services/uploads/README.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/services/uploads/README.md b/services/uploads/README.md
index 62cce27611..2df913f1a2 100644
--- a/services/uploads/README.md
+++ b/services/uploads/README.md
@@ -16,6 +16,32 @@ In addition to live scanning all uploaded files, we also have a pair of lambdas
 
 The avAuditUploads lambda pulls a list of every file in the uploads bucket and then invokes avAuditFiles for each of them in chunks of 20 or so. For any files that are found to be INFECTED, it then grabs the current s3 tags for them and verifies that they are tagged accordingly. If not, it will re-tag the file to be INFECTED, preventing further download.
 
+## ClamAV Daemon
+
+We have an ec2 instance that is created in dev, val, and prod that is configured with an always on ClamAV instance that accepts incoming virus scan requests on port 3310. The motivation here is that we are working towards having our av scanning lambdas use the always on ClamAV daemon rather than rely on just the lambda, as there is a high startup cost for ClamAV of around 29 seconds in our testing. This means that all virus scans take at least 29 seconds for the user. By using an always on instance we can reduce that to closer to the actual time of the virus scan (usually < 1s).
+
+The server is restricted to only have access from connections in the default security group as well as anything else that is placed in the ClamAV security group. This allows for our AV scanning lambds to call out to the instance while restricting all other traffic. However, all of our engineers have ssh pub keys on the instance in case they need access to the machine via ssh for any reason.
+
+### Accessing the VM
+
+Similar to the [Postgres jumpbox](../postgres/README.md), we use the `authorized_keys` file to give access to this VM and you'll need to add your IP to the VM's security group:
+
+1. Determine your public facing IP address. An easy way to do this is to `curl https://ifconfig.me/`
+2. Locate the EC2 instance in the AWS console. Click and go into Security > Security groups.
+3. There should be two security groups attached to the instance, the default and the ClamAV one. Select the ClamAV security group.
+4. On the `Inbound rules` tab select `Edit inbound rules`
+5. Add a rule for `ssh` with the `source` set to your local IP address with `/32` appended to it (e.g. `1.2.3.4/32`)
+6. Save the rule
+
+#### SSH to the instances
+
+You should now be able to ssh to the jump box.
+
+1. Locate the Public IPv4 address of the instance. This can be found by clicking into the VM on the `Instances` section of the EC2 console.
+2. ssh ubuntu@public-ip
+
+You should be using public key auth to ssh. If you need to point to your private key, use `ssh -i ~/.ssh/${yourkeyfile} ubuntu@public-ip`
+
 ## Significant dependencies
 
 -   serverless-s3-upload

From 3be571cdb4e243a185ff08c52f9da21b0d126460 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 8 Feb 2024 15:32:38 -0500
Subject: [PATCH 15/38] add clamd.conf to the layer

---
 services/uploads/src/avLayer/build/build.sh   |  1 +
 services/uploads/src/avLayer/build/clamd.conf | 12 ++++++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 services/uploads/src/avLayer/build/clamd.conf

diff --git a/services/uploads/src/avLayer/build/build.sh b/services/uploads/src/avLayer/build/build.sh
index addc74f5c5..0003345419 100755
--- a/services/uploads/src/avLayer/build/build.sh
+++ b/services/uploads/src/avLayer/build/build.sh
@@ -51,6 +51,7 @@ cp /tmp/build/usr/local/bin/clamscan /tmp/build/usr/local/bin/clamdscan /tmp/bui
 cp -R /tmp/build/usr/lib64/* lib/.
 cp -R /tmp/build/usr/local/lib64/* lib/.
 cp freshclam.conf bin/freshclam.conf
+cp clamd.conf bin/clamd.conf
 
 zip -r9 lambda_layer.zip bin
 zip -r9 lambda_layer.zip lib
\ No newline at end of file
diff --git a/services/uploads/src/avLayer/build/clamd.conf b/services/uploads/src/avLayer/build/clamd.conf
new file mode 100644
index 0000000000..147062cab6
--- /dev/null
+++ b/services/uploads/src/avLayer/build/clamd.conf
@@ -0,0 +1,12 @@
+# hostname and port of the remote ClamAV daemon
+TCPAddr ip-10-147-219-140.ec2.internal
+TCPSocket  3310
+
+# Enable verbose logging
+LogVerbose yes
+
+# Path to the log file 
+LogFile /var/log/clamd.log
+
+# Set the maximum number of concurrent threads for scanning
+MaxThreads  10
\ No newline at end of file

From c2fa0c3ce869fe0a7829a865ceada780ccffb66d Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 8 Feb 2024 16:30:01 -0500
Subject: [PATCH 16/38] bump lambda layer

---
 services/uploads/serverless.yml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 9de2d1a42c..c60b985b7d 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -58,15 +58,6 @@ custom:
     packager: yarn
     packagerOptions:
       lockFile: ../../yarn.lock
-  scripts:
-    hooks:
-      # This script is run locally when running 'serverless deploy'
-      package:initialize: |
-        set -e
-        curl -L --output lambda_layer.zip https://github.com/CMSgov/lambda-clamav-layer/releases/download/0.7/lambda_layer.zip
-      deploy:finalize: |
-        rm lambda_layer.zip
-        serverless invoke --stage ${sls:stage} --function avDownloadDefinitions -t Event
   vpcId: ${ssm:/configuration/${sls:stage}/vpc/id, ssm:/configuration/default/vpc/id}
   sgId: ${ssm:/configuration/${sls:stage}/vpc/sg/id, ssm:/configuration/default/vpc/sg/id}
   privateSubnets:
@@ -104,8 +95,7 @@ custom:
 
 layers:
   clamAv:
-    package:
-      artifact: lambda_layer.zip
+    path: lambda_layer
 
 functions:
   avScan:

From a639d25abecc0d376e32d55a84fc10c23f59b948 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 8 Feb 2024 17:05:59 -0500
Subject: [PATCH 17/38] fix path name

---
 services/uploads/serverless.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index c60b985b7d..3c93c280ed 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -95,7 +95,7 @@ custom:
 
 layers:
   clamAv:
-    path: lambda_layer
+    path: lambda-layers-clamav
 
 functions:
   avScan:

From 63a308a98183f72455942a9696322456f56102e8 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 9 Feb 2024 15:43:40 -0500
Subject: [PATCH 18/38] do the scan remote

---
 services/uploads/src/deps/clamAV/clamAV.ts | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/services/uploads/src/deps/clamAV/clamAV.ts b/services/uploads/src/deps/clamAV/clamAV.ts
index da3ee29aae..7e916192a7 100644
--- a/services/uploads/src/deps/clamAV/clamAV.ts
+++ b/services/uploads/src/deps/clamAV/clamAV.ts
@@ -177,6 +177,16 @@ function scanForInfectedFiles(
     try {
         console.info('Executing clamav')
 
+        // try using clamdscan as a test on this branch
+        const avResult = spawnSync('/opt/bin/clamdscan', [
+            '--stdout',
+            '-v',
+            '--config-file=/opt/bin/clamd.conf',
+            '--stream',
+            pathToScan,
+        ])
+
+        /*
         const avResult = spawnSync(config.pathToClamav, [
             '--stdout',
             '-v',
@@ -184,6 +194,7 @@ function scanForInfectedFiles(
             config.pathToDefintions,
             pathToScan,
         ])
+        */
 
         console.info('stderror', avResult.stderr && avResult.stderr.toString())
         console.info('stdout', avResult.stdout && avResult.stdout.toString())

From 814d621b0dd30b60e249316a0c8d7ad68e0a0ba8 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 9 Feb 2024 16:35:17 -0500
Subject: [PATCH 19/38] drop the wait just to see

---
 services/app-web/src/s3/s3Amplify.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts
index 0b5fa71bbd..9376ed9dd0 100644
--- a/services/app-web/src/s3/s3Amplify.ts
+++ b/services/app-web/src/s3/s3Amplify.ts
@@ -108,7 +108,7 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT {
             bucket: BucketShortName
         ): Promise<void | S3Error> => {
             try {
-                await waitFor(20000)
+                await waitFor(5000)
                 try {
                     await retryWithBackoff(async () => {
                         await Storage.get(filename, {

From cec7e762230e71a327bac61358c8b2a91b8a1b00 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 9 Feb 2024 17:02:06 -0500
Subject: [PATCH 20/38] let's do it live!

---
 services/app-web/src/s3/s3Amplify.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts
index 9376ed9dd0..3c89fc0cf0 100644
--- a/services/app-web/src/s3/s3Amplify.ts
+++ b/services/app-web/src/s3/s3Amplify.ts
@@ -108,7 +108,6 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT {
             bucket: BucketShortName
         ): Promise<void | S3Error> => {
             try {
-                await waitFor(5000)
                 try {
                     await retryWithBackoff(async () => {
                         await Storage.get(filename, {

From 11d7551314c5f625369f1e87f714c19e764c9f6d Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Fri, 9 Feb 2024 19:31:26 -0500
Subject: [PATCH 21/38] some is better than none here I guess

---
 services/app-web/src/s3/s3Amplify.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts
index 3c89fc0cf0..b5c2c5ac1d 100644
--- a/services/app-web/src/s3/s3Amplify.ts
+++ b/services/app-web/src/s3/s3Amplify.ts
@@ -108,6 +108,7 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT {
             bucket: BucketShortName
         ): Promise<void | S3Error> => {
             try {
+                await waitFor(3000)
                 try {
                     await retryWithBackoff(async () => {
                         await Storage.get(filename, {

From 53643c1c2a65aacca391b87a44695c69a07c8b9e Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 12 Feb 2024 15:58:54 -0500
Subject: [PATCH 22/38] add route53 internal DNS

---
 services/uploads/serverless.yml               | 20 ++++++++++++++++++-
 services/uploads/src/avLayer/build/clamd.conf |  3 ++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 3c93c280ed..2b7af0a100 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -422,7 +422,6 @@ resources:
         InstanceType: t3.medium
         ImageId: ami-0c7217cdde317cfec # Ubuntu 22.04 LTS
         IamInstanceProfile: !Ref ClamAVInstanceProfile
-        KeyName: mojo # for testing
         NetworkInterfaces:
           - AssociatePublicIpAddress: true
             DeviceIndex: '0'
@@ -460,6 +459,25 @@ resources:
             systemctl enable clamav-daemon
             systemctl start clamav-daemon
 
+  MCRInternalZone:
+    Type: AWS::Route53::HostedZone
+    DependsOn: ClamAVInstance
+    Properties:
+      Name: mc-review.local
+      VPCs:
+        - VPCId: vpc-0b59998530ab3d795 # for now, testing
+          VPCRegion: !Ref AWS::Region
+
+  ClamAVRecordSet:
+    Type: AWS::Route53::RecordSet
+    Properties:
+      HostedZoneId: !Ref ClamAVInternalZone
+      Name: clamav.mc-review.local
+      Type: A
+      ResourceRecords:
+        - !GetAtt ClamAVInstance.PrivateIp
+      TTL: '300'
+
   Outputs:
     DocumentUploadsBucketName:
       Value: !Ref DocumentUploadsBucket
diff --git a/services/uploads/src/avLayer/build/clamd.conf b/services/uploads/src/avLayer/build/clamd.conf
index 147062cab6..acd9fd34ef 100644
--- a/services/uploads/src/avLayer/build/clamd.conf
+++ b/services/uploads/src/avLayer/build/clamd.conf
@@ -1,5 +1,6 @@
 # hostname and port of the remote ClamAV daemon
-TCPAddr ip-10-147-219-140.ec2.internal
+# TCPAddr ip-10-147-219-140.ec2.internal
+TCPAddr clamav.mc-review.local
 TCPSocket  3310
 
 # Enable verbose logging

From be82dfeaa5f5102dc1e97aa7ca743c585e8403d0 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 12 Feb 2024 17:18:46 -0500
Subject: [PATCH 23/38] add test dns server

---
 services/uploads/serverless.yml | 38 +++++++++++++++++----------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 2b7af0a100..1665af8b47 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -177,7 +177,7 @@ resources:
       - !Equals ['${sls:stage}', 'main']
       - !Equals ['${sls:stage}', 'val']
       - !Equals ['${sls:stage}', 'prod']
-      - !Equals ['${sls:stage}', 'mtbuildclamavserver']
+      - !Equals ['${sls:stage}', 'mtscanfromlambda']
 
   Resources:
     DocumentUploadsBucket:
@@ -459,24 +459,26 @@ resources:
             systemctl enable clamav-daemon
             systemctl start clamav-daemon
 
-  MCRInternalZone:
-    Type: AWS::Route53::HostedZone
-    DependsOn: ClamAVInstance
-    Properties:
-      Name: mc-review.local
-      VPCs:
-        - VPCId: vpc-0b59998530ab3d795 # for now, testing
-          VPCRegion: !Ref AWS::Region
+    MCRInternalZone:
+      Type: AWS::Route53::HostedZone
+      Condition: IsDevValProd
+      Properties:
+        Name: mc-review.local
+        VPCs:
+          - VPCId: vpc-0b59998530ab3d795 # for now, testing
+            VPCRegion: !Ref AWS::Region
 
-  ClamAVRecordSet:
-    Type: AWS::Route53::RecordSet
-    Properties:
-      HostedZoneId: !Ref ClamAVInternalZone
-      Name: clamav.mc-review.local
-      Type: A
-      ResourceRecords:
-        - !GetAtt ClamAVInstance.PrivateIp
-      TTL: '300'
+    ClamAVRecordSet:
+      Type: AWS::Route53::RecordSet
+      Condition: IsDevValProd
+      DependsOn: ClamAVInstance
+      Properties:
+        HostedZoneId: !Ref MCRInternalZone
+        Name: clamav.mc-review.local
+        Type: A
+        ResourceRecords:
+          - !GetAtt ClamAVInstance.PrivateIp
+        TTL: '300'
 
   Outputs:
     DocumentUploadsBucketName:

From 1742375db9987bc3d7f306ceec3b796c142c271e Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Mon, 12 Feb 2024 17:53:38 -0500
Subject: [PATCH 24/38] thread the vpcid

---
 services/uploads/serverless.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 1665af8b47..a18cc52517 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -465,7 +465,7 @@ resources:
       Properties:
         Name: mc-review.local
         VPCs:
-          - VPCId: vpc-0b59998530ab3d795 # for now, testing
+          - VPCId: ${self:custom.vpcId}
             VPCRegion: !Ref AWS::Region
 
     ClamAVRecordSet:

From 7ae55a47d95d1a4105d179f18c0b0c2485d4198a Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 12:11:58 -0500
Subject: [PATCH 25/38] cleanup apt usage

---
 services/uploads/serverless.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index a18cc52517..ff773013ef 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -438,7 +438,6 @@ resources:
           Fn::Base64: !Sub |
             #!/bin/bash
             apt-get update
-            apt-get install -y clamav clamav-daemon
 
             echo '${self:custom.authorizedKeys}' > /home/ubuntu/.ssh/authorized_keys
             chown ubuntu:ubuntu /home/ubuntu/.ssh/authorized_keys

From 96170940710017f88fe4e5ec957433b1ff85a789 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 15:42:18 -0500
Subject: [PATCH 26/38] revert

---
 services/uploads/serverless.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index ff773013ef..a18cc52517 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -438,6 +438,7 @@ resources:
           Fn::Base64: !Sub |
             #!/bin/bash
             apt-get update
+            apt-get install -y clamav clamav-daemon
 
             echo '${self:custom.authorizedKeys}' > /home/ubuntu/.ssh/authorized_keys
             chown ubuntu:ubuntu /home/ubuntu/.ssh/authorized_keys

From 68c231cf224a1dff1389d4ab8a7e80d449cebdd7 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 16:04:44 -0500
Subject: [PATCH 27/38] swap order. clean

---
 services/uploads/serverless.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index a18cc52517..3a57b4769d 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -444,20 +444,15 @@ resources:
             chown ubuntu:ubuntu /home/ubuntu/.ssh/authorized_keys
             chmod 600 /home/ubuntu/.ssh/authorized_keys
 
-            # Install clamav 1.0.4
-            wget https://www.clamav.net/downloads/production/clamav-1.0.4.linux.x86_64.deb
-            dpkg -i clamav-1.0.4.linux.x86_64.deb
-            apt-get install -f
-
             # Write to the clamd.conf
             echo "TCPSocket 3310" >> /etc/clamav/clamd.conf
             echo "TCPAddr 0.0.0.0" >> /etc/clamav/clamd.conf 
 
             # Start clamd and get defs
-            systemctl enable clamav-freshclam
-            systemctl start clamav-freshclam
             systemctl enable clamav-daemon
+            systemctl enable clamav-freshclam
             systemctl start clamav-daemon
+            systemctl start clamav-freshclam
 
     MCRInternalZone:
       Type: AWS::Route53::HostedZone

From d57f3e1488c9754f60bfa18d6424ddd4937ab9e7 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 16:08:17 -0500
Subject: [PATCH 28/38] confirmation. remove comment.

---
 services/uploads/serverless.yml               | 4 ++++
 services/uploads/src/avLayer/build/clamd.conf | 1 -
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 3a57b4769d..c2b60ec954 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -454,6 +454,10 @@ resources:
             systemctl start clamav-daemon
             systemctl start clamav-freshclam
 
+            # Confirm we're up
+            systemctl status clamav-daemon
+            systemctl status clamav-freshclam
+
     MCRInternalZone:
       Type: AWS::Route53::HostedZone
       Condition: IsDevValProd
diff --git a/services/uploads/src/avLayer/build/clamd.conf b/services/uploads/src/avLayer/build/clamd.conf
index acd9fd34ef..0ac6deeaba 100644
--- a/services/uploads/src/avLayer/build/clamd.conf
+++ b/services/uploads/src/avLayer/build/clamd.conf
@@ -1,5 +1,4 @@
 # hostname and port of the remote ClamAV daemon
-# TCPAddr ip-10-147-219-140.ec2.internal
 TCPAddr clamav.mc-review.local
 TCPSocket  3310
 

From ef91850a8643b13da57b2afe751fc36b9ecb58ac Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 16:12:44 -0500
Subject: [PATCH 29/38] use clamdscan for real

---
 services/uploads/src/deps/clamAV/clamAV.ts | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/services/uploads/src/deps/clamAV/clamAV.ts b/services/uploads/src/deps/clamAV/clamAV.ts
index 7e916192a7..f746615383 100644
--- a/services/uploads/src/deps/clamAV/clamAV.ts
+++ b/services/uploads/src/deps/clamAV/clamAV.ts
@@ -32,9 +32,9 @@ function NewClamAV(config: Partial<ClamAVConfig>, s3Client: S3UploadsClient) {
         bucketName: config.bucketName,
         definitionsPath: config.definitionsPath,
 
-        pathToClamav: config.pathToClamav || '/opt/bin/clamscan',
+        pathToClamav: config.pathToClamav || '/opt/bin/clamdscan',
         pathToFreshclam: config.pathToFreshclam || '/opt/bin/freshclam',
-        pathToConfig: config.pathToConfig || '/opt/bin/freshclam.conf',
+        pathToConfig: config.pathToConfig || '/opt/bin/clamd.conf',
         pathToDefintions: config.pathToDefintions || '/tmp',
     }
 
@@ -177,11 +177,11 @@ function scanForInfectedFiles(
     try {
         console.info('Executing clamav')
 
-        // try using clamdscan as a test on this branch
-        const avResult = spawnSync('/opt/bin/clamdscan', [
+        // use clamdscan to connect to our clamavd server
+        const avResult = spawnSync(config.pathToClamav, [
             '--stdout',
             '-v',
-            '--config-file=/opt/bin/clamd.conf',
+            `--config-file=${config.pathToConfig}`,
             '--stream',
             pathToScan,
         ])

From 19d50574b8f8a2c53a08e5d69061e0251ca0cfb6 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 16:39:57 -0500
Subject: [PATCH 30/38] differentiate clamdscan from clamscan

---
 services/uploads/src/deps/clamAV/clamAV.ts | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/services/uploads/src/deps/clamAV/clamAV.ts b/services/uploads/src/deps/clamAV/clamAV.ts
index f746615383..b50cddceed 100644
--- a/services/uploads/src/deps/clamAV/clamAV.ts
+++ b/services/uploads/src/deps/clamAV/clamAV.ts
@@ -21,6 +21,9 @@ interface ClamAVConfig {
     pathToFreshclam: string
     pathToConfig: string
     pathToDefintions: string
+
+    pathToClamdScan: string
+    pathToClamdConfig: string
 }
 
 function NewClamAV(config: Partial<ClamAVConfig>, s3Client: S3UploadsClient) {
@@ -32,10 +35,13 @@ function NewClamAV(config: Partial<ClamAVConfig>, s3Client: S3UploadsClient) {
         bucketName: config.bucketName,
         definitionsPath: config.definitionsPath,
 
-        pathToClamav: config.pathToClamav || '/opt/bin/clamdscan',
+        pathToClamav: config.pathToClamav || '/opt/bin/clamscan',
         pathToFreshclam: config.pathToFreshclam || '/opt/bin/freshclam',
-        pathToConfig: config.pathToConfig || '/opt/bin/clamd.conf',
+        pathToConfig: config.pathToConfig || '/opt/bin/freshclam.conf',
         pathToDefintions: config.pathToDefintions || '/tmp',
+
+        pathToClamdScan: config.pathToClamdScan || '/opt/bin/clamdscan',
+        pathToClamdConfig: config.pathToClamdConfig || '/opt/bin/clamd.conf',
     }
 
     return {
@@ -178,10 +184,10 @@ function scanForInfectedFiles(
         console.info('Executing clamav')
 
         // use clamdscan to connect to our clamavd server
-        const avResult = spawnSync(config.pathToClamav, [
+        const avResult = spawnSync(config.pathToClamdScan, [
             '--stdout',
             '-v',
-            `--config-file=${config.pathToConfig}`,
+            `--config-file=${config.pathToClamdConfig}`,
             '--stream',
             pathToScan,
         ])

From fc51858ed7d52b14ceeccf2f13929612947aaa97 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 17:06:15 -0500
Subject: [PATCH 31/38] add route53 to gh-oidc for ci

---
 services/github-oidc/serverless.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/github-oidc/serverless.yml b/services/github-oidc/serverless.yml
index 0cfbbb0e63..2f6357ab87 100644
--- a/services/github-oidc/serverless.yml
+++ b/services/github-oidc/serverless.yml
@@ -53,6 +53,7 @@ params:
       - 'kms:*'
       - 'lambda:*'
       - 'logs:*'
+      - 'route53:*'
       - 'rds:*'
       - 'secretsmanager:*'
       - 'ssm:*'

From ccb8f0920f84654ac83070a144160a9043d50a2d Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 17:45:12 -0500
Subject: [PATCH 32/38] no need to fetch defs in this world

---
 services/uploads/src/lib/scanFiles.ts | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/services/uploads/src/lib/scanFiles.ts b/services/uploads/src/lib/scanFiles.ts
index de1587cfe4..6723eb8f27 100644
--- a/services/uploads/src/lib/scanFiles.ts
+++ b/services/uploads/src/lib/scanFiles.ts
@@ -12,14 +12,6 @@ export async function scanFiles(
     bucket: string,
     scanDir: string
 ): Promise<string[] | Error> {
-    // fetch definition files
-    console.info('Download AV Definitions')
-    const defsRes = await clamAV.downloadAVDefinitions()
-    if (defsRes) {
-        console.error('failed to fetch definitions')
-        return defsRes
-    }
-
     // clamScan wants files to be top level in the scanned directory, so we map each key to a UUID
     const filemap: { [filename: string]: string } = {}
     for (const key of keys) {

From 8198b4b05b192df3a297cdcd25302e6e6b80b13d Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Tue, 13 Feb 2024 18:23:32 -0500
Subject: [PATCH 33/38] fix the userdata script up a bit

---
 services/uploads/serverless.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index c2b60ec954..a06a55f6f4 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -448,6 +448,18 @@ resources:
             echo "TCPSocket 3310" >> /etc/clamav/clamd.conf
             echo "TCPAddr 0.0.0.0" >> /etc/clamav/clamd.conf 
 
+            # Create a systemd service override to delay the start
+            cat <<EOF > /etc/systemd/system/clamav-daemon.service.d/override.conf
+            [Unit]
+            After=network.target
+            EOF
+
+            # Fix the systemctl setting
+            sed -i 's/^StandardOutput=syslog/StandardOutput=journal/' /lib/systemd/system/clamav-daemon.service
+
+            # Reload systemd to apply the changes
+            systemctl daemon-reload
+
             # Start clamd and get defs
             systemctl enable clamav-daemon
             systemctl enable clamav-freshclam

From 2765110ad615c2a5964491913b68c54e54db9b7b Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Wed, 14 Feb 2024 11:21:24 -0500
Subject: [PATCH 34/38] return faster for positive virus scan

---
 services/app-web/src/s3/s3Amplify.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts
index b5c2c5ac1d..bf629d3477 100644
--- a/services/app-web/src/s3/s3Amplify.ts
+++ b/services/app-web/src/s3/s3Amplify.ts
@@ -99,8 +99,8 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT {
         },
         /*  
             Poll for scanning completion
-            - We start polling after 20s, which is the estimated time it takes scanning to start to resolve.
-            - In total, each file could be up to 40 sec in a loading state (20s wait for scanning + 8s of retries + extra time for uploading and scanning api requests to resolve)
+            - We start polling after 3s, which is the estimated time it takes scanning to start to resolve.
+            - We then retry with an exponential backoff for up to 15s. Most scans take < 1s, plus some additional time for tagging and response, so by 15s a file should be tagged.
             - While the file is scanning, returns 403. When scanning is complete, the resource returns 200
         */
         scanFile: async (
@@ -213,7 +213,7 @@ const waitFor = (delay = 1000) =>
 const retryWithBackoff = async (
     fn: () => Promise<void | S3Error>,
     retryCount = 0,
-    maxRetries = 6,
+    maxRetries = 4,
     err: null | S3Error = null
 ): Promise<void | S3Error> => {
     if (retryCount > maxRetries) {

From cbe6c8dc1fa391c766f83913369a3b57f26e537c Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Wed, 14 Feb 2024 16:43:06 -0500
Subject: [PATCH 35/38] drop default sg

---
 services/uploads/serverless.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index a06a55f6f4..17d2f23e7a 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -108,7 +108,6 @@ functions:
       - !Ref ClamAvLambdaLayer
       - arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:1
     vpc:
-      securityGroupIds: ${self:custom.sgId}
       subnetIds: ${self:custom.privateSubnets}
     environment:
       stage: ${sls:stage}
@@ -427,7 +426,6 @@ resources:
             DeviceIndex: '0'
             GroupSet:
               - !Ref ClamAVSecurityGroup
-              - ${self:custom.sgId}
             SubnetId: !Sub ${self:custom.publicSubnetA}
         Tags:
           - Key: Name

From a975a966aaf7f2b17b7bc82c59470dd00d5c0ff3 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Wed, 14 Feb 2024 17:23:17 -0500
Subject: [PATCH 36/38] github-oidc isn't running here

---
 .github/workflows/deploy-infra-to-env.yml | 2 +-
 .github/workflows/deploy.yml              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-infra-to-env.yml b/.github/workflows/deploy-infra-to-env.yml
index 00ee028f2b..a09a0e7ae0 100644
--- a/.github/workflows/deploy-infra-to-env.yml
+++ b/.github/workflows/deploy-infra-to-env.yml
@@ -29,7 +29,7 @@ permissions:
 jobs:
   # deploy the OIDC service first, as subsequent deploys may depend on updated OIDC permissions
   github-oidc:
-    if: ${{ contains(inputs.changed_services, 'github-oidc') }}
+    if: ${{ contains(inputs.changed_services, 'github-oidc') || contains(inputs.changed_services, 'uploads') }}
     environment: ${{ inputs.environment }}
     runs-on: ubuntu-20.04
     steps:
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index ce92b60b9b..b784b6ba12 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -272,7 +272,7 @@ jobs:
 
   deploy-infra:
     needs: [begin-deployment, build-clamav-layer]
-    uses: Enterprise-CMCS/managed-care-review/.github/workflows/deploy-infra-to-env.yml@main
+    uses: Enterprise-CMCS/managed-care-review/.github/workflows/deploy-infra-to-env.yml@mt-scan-from-lambda
     with:
       environment: dev
       stage_name: ${{ needs.begin-deployment.outputs.stage-name}}

From 1f697199f93b9bf0785ff01efaa57faf3c6608bf Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Wed, 14 Feb 2024 18:20:05 -0500
Subject: [PATCH 37/38] revert

---
 .github/workflows/deploy-infra-to-env.yml | 2 +-
 .github/workflows/deploy.yml              | 2 +-
 services/uploads/serverless.yml           | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-infra-to-env.yml b/.github/workflows/deploy-infra-to-env.yml
index a09a0e7ae0..00ee028f2b 100644
--- a/.github/workflows/deploy-infra-to-env.yml
+++ b/.github/workflows/deploy-infra-to-env.yml
@@ -29,7 +29,7 @@ permissions:
 jobs:
   # deploy the OIDC service first, as subsequent deploys may depend on updated OIDC permissions
   github-oidc:
-    if: ${{ contains(inputs.changed_services, 'github-oidc') || contains(inputs.changed_services, 'uploads') }}
+    if: ${{ contains(inputs.changed_services, 'github-oidc') }}
     environment: ${{ inputs.environment }}
     runs-on: ubuntu-20.04
     steps:
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index b784b6ba12..ce92b60b9b 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -272,7 +272,7 @@ jobs:
 
   deploy-infra:
     needs: [begin-deployment, build-clamav-layer]
-    uses: Enterprise-CMCS/managed-care-review/.github/workflows/deploy-infra-to-env.yml@mt-scan-from-lambda
+    uses: Enterprise-CMCS/managed-care-review/.github/workflows/deploy-infra-to-env.yml@main
     with:
       environment: dev
       stage_name: ${{ needs.begin-deployment.outputs.stage-name}}
diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 17d2f23e7a..8eb436a061 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -108,6 +108,7 @@ functions:
       - !Ref ClamAvLambdaLayer
       - arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:1
     vpc:
+      securityGroupIds: ${self:custom.sgId}
       subnetIds: ${self:custom.privateSubnets}
     environment:
       stage: ${sls:stage}

From 1990611028fa41c9e1e8c75aa18ecbff55654dd9 Mon Sep 17 00:00:00 2001
From: Mojo Talantikite <mojo.talantikite@gmail.com>
Date: Thu, 15 Feb 2024 11:17:54 -0500
Subject: [PATCH 38/38] restart if failed to start at boot

---
 services/uploads/serverless.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml
index 8eb436a061..eb8002c946 100644
--- a/services/uploads/serverless.yml
+++ b/services/uploads/serverless.yml
@@ -453,6 +453,14 @@ resources:
             After=network.target
             EOF
 
+            # Create a systemd service override to delay the start and set restart limits
+            cat <<EOF > /etc/systemd/system/clamav-daemon.service.d/override.conf
+            [Unit]
+            After=network.target
+            StartLimitIntervalSec=1h
+            StartLimitBurst=5
+            EOF
+
             # Fix the systemctl setting
             sed -i 's/^StandardOutput=syslog/StandardOutput=journal/' /lib/systemd/system/clamav-daemon.service