feat: Add Cleaner (#106)

.github/workflows/cron.yml

@@ -6,8 +6,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  run-cron:
-    name: "Run Cron"
+  run-report:
+    name: "Run Report"
     runs-on: ubuntu-22.04
     permissions:
       id-token: write   # This is required for requesting the JWT
@@ -33,3 +33,27 @@ jobs:
         env:
           TO_EMAIL: ${{ secrets.TO_EMAIL }}
           FROM_EMAIL: ${{ secrets.FROM_EMAIL }}
+  run-cleaner:
+    name: "Run Cleaner"
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write   # This is required for requesting the JWT
+      contents: read    # This is required for actions/checkout
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 6.0.x
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
+          aws-region: "ap-southeast-1"
+      - name: Restore dependencies
+        run: dotnet restore
+      - name: Build
+        run: dotnet build --no-restore
+      - name: Test
+        run: dotnet run --no-build --project ServiceMonitor.Cleaner

ServiceMonitor.AWS/ECR.cs

@@ -8,16 +8,22 @@
 using Amazon.ECR;
 using Amazon.ECR.Model;
 using System.Linq;
+using ServiceMonitor.Cloud.Model;
+using Microsoft.Extensions.Logging;
 
 namespace ServiceMonitor.AWS
 {
     public class ECR : IImage
     {
         private readonly IServiceProvider _serviceProvider;
-        public ECR(IServiceProvider serviceProvider)
+        private readonly ILogger<ECR> _logger;
+
+        public ECR(IServiceProvider serviceProvider, ILogger<ECR> logger)
         {
             _serviceProvider = serviceProvider;
+            _logger = logger;
         }
+
         public async Task<ICollection<BasicProperty>> GetImagesAsync(string region, CancellationToken cancellationToken = default)
         {
             // TODO: need better solution for client resolution
@@ -43,5 +49,91 @@ public async Task<ICollection<BasicProperty>> GetImagesAsync(string region, Canc
 
             return list;
         }
+
+        public async Task<ICollection<TagItem>> GetTags(string region, string repoName, CancellationToken cancellationToken = default)
+        {
+            var ecrRegion = RegionEndpoint.GetBySystemName(region);
+            using var ecrClient = ActivatorUtilities.CreateInstance<AmazonECRClient>(_serviceProvider, ecrRegion);
+            if (ecrClient == null)
+            {
+                throw new ArgumentNullException(nameof(ecrClient));
+            }
+            var imageListRequest = new DescribeImagesRequest
+            {
+                RepositoryName = repoName,
+                Filter = new Amazon.ECR.Model.DescribeImagesFilter
+                {
+                    TagStatus = TagStatus.TAGGED,
+                }
+            };
+            var tagsResources = await ecrClient.DescribeImagesAsync(imageListRequest);
+            return tagsResources.ImageDetails.Select(x => new TagItem
+            {
+                CreatedTime = x.ImagePushedAt,
+                Tags = x.ImageTags,
+                ImageDigest = x.ImageDigest,
+            }).ToList();
+        }
+
+        public async Task<(List<string>, List<string>)> DeleteTags(string region, string repoName, ICollection<TagItem> tags, CancellationToken cancellationToken = default)
+        {
+            var sample = tags.FirstOrDefault();
+            if (sample == null)
+            {
+                return (new List<string>(), new List<string>());
+            }
+
+            var ecrRegion = RegionEndpoint.GetBySystemName(region);
+            using var ecrClient = ActivatorUtilities.CreateInstance<AmazonECRClient>(_serviceProvider, ecrRegion);
+            if (ecrClient == null)
+            {
+                throw new ArgumentNullException(nameof(ecrClient));
+            }
+
+            var isShaMode = sample.Tags.Contains("sha-");
+            var deleteImages = new List<ImageIdentifier>();
+            if (isShaMode)
+            {
+                foreach (var tag in tags)
+                {
+                    if (tag.Tags.Contains("main") || tag.Tags.Contains("latest") || tag.Tags.Contains("master"))
+                    {
+                        continue;
+                    }
+                    deleteImages.Add(new ImageIdentifier
+                    {
+                        ImageDigest = tag.ImageDigest,
+                    });
+                }
+            }
+            else
+            {
+                var sorted = tags.OrderByDescending(x => x.CreatedTime).Select(x => new ImageIdentifier
+                {
+                    ImageDigest = x.ImageDigest,
+                }).ToList();
+                sorted.RemoveAt(0);
+                deleteImages.AddRange(sorted);
+            }
+
+            if (deleteImages.Count == 0)
+            {
+                return (new List<string>(), new List<string>());
+            }
+
+            _logger.LogInformation("Will remove {} tags", deleteImages.Count);
+
+            var deleteImageRequest = new BatchDeleteImageRequest
+            {
+                RepositoryName = repoName,
+                ImageIds = deleteImages,
+            };
+
+            var batchResponse = await ecrClient.BatchDeleteImageAsync(deleteImageRequest);
+
+            _logger.LogInformation("Success remove {} and fail {}", batchResponse.ImageIds.Count, batchResponse.Failures.Count);
+
+            return (batchResponse.ImageIds.Select(x => x.ImageTag).ToList(), batchResponse.Failures.Select(x => x.ImageId.ImageTag).ToList());
+        }
     }
 }

ServiceMonitor.AWS/ServiceMonitor.AWS.csproj

@@ -11,6 +11,7 @@
     <PackageReference Include="AWSSDK.Lambda" Version="3.7.201.22" />
     <PackageReference Include="AWSSDK.S3" Version="3.7.203.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>

ServiceMonitor.Cleaner/Program.cs

@@ -0,0 +1,64 @@
+
+using Amazon;
+using Amazon.EC2.Model;
+using Amazon.SimpleEmailV2;
+using Amazon.SimpleEmailV2.Model;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using ServiceMonitor.AWS;
+using ServiceMonitor.Cloud;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+IConfiguration config = new ConfigurationBuilder()
+    .SetBasePath(Directory.GetCurrentDirectory())
+    .AddJsonFile("appsettings.json", true)
+    .AddEnvironmentVariables()
+    .Build();
+
+var serviceCollection = new ServiceCollection();
+serviceCollection.AddSingleton(config);
+serviceCollection.AddLogging(configure =>
+{
+    configure.ClearProviders();
+    configure.AddConfiguration(config.GetSection("Logging"));
+    configure.AddConsole();
+});
+serviceCollection.AddAWSService<IAmazonSimpleEmailServiceV2>();
+serviceCollection.AddScoped<IAppRunner, AppRunner>();
+serviceCollection.AddScoped<IInstance, AWSInstance>();
+serviceCollection.AddScoped<IFunction, Lambda>();
+serviceCollection.AddScoped<IImage, ECR>();
+serviceCollection.AddScoped<IBucket, S3>();
+
+var serviceProvider = serviceCollection.BuildServiceProvider();
+
+var logger = serviceProvider.GetRequiredService<ILoggerFactory>().CreateLogger<Program>();
+logger.LogDebug("Starting application");
+
+var regions = new List<string>
+{
+    "ap-southeast-1",
+    "ap-southeast-3",
+};
+
+var containerImages = serviceProvider.GetRequiredService<IImage>();
+foreach (var region in regions)
+{
+    var images = await containerImages.GetImagesAsync(region);
+    foreach (var image in images)
+    {
+        var tags = await containerImages.GetTags(region, image.Name);
+        var (success, failure) = await containerImages.DeleteTags(region, image.Name, tags);
+        logger.LogInformation("{}:{}:{}", image.Name, JsonSerializer.Serialize(success, new JsonSerializerOptions
+        {
+            WriteIndented = true,
+        }),
+         JsonSerializer.Serialize(failure, new JsonSerializerOptions
+         {
+             WriteIndented = true,
+         }));
+    }
+}

ServiceMonitor.Cleaner/ServiceMonitor.Cleaner.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.Extensions.NETCore.Setup" Version="3.7.7" />
+    <PackageReference Include="AWSSDK.SimpleEmailV2" Version="3.7.200.26" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ServiceMonitor.AWS\ServiceMonitor.AWS.csproj" />
+  </ItemGroup>
+
+</Project>

ServiceMonitor.Cloud/IImage.cs

@@ -1,11 +1,14 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using System.Threading;
+using ServiceMonitor.Cloud.Model;
 
 namespace ServiceMonitor.Cloud
 {
     public interface IImage
     {
         Task<ICollection<BasicProperty>> GetImagesAsync(string region, CancellationToken cancellationToken = default);
+        Task<ICollection<TagItem>> GetTags(string region, string repoName, CancellationToken cancellationToken = default);
+        Task<(List<string>, List<string>)> DeleteTags(string region, string repoName, ICollection<TagItem> tags, CancellationToken cancellationToken = default);
     }
 }

ServiceMonitor.Cloud/Model/TagItem.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Text;
+
+namespace ServiceMonitor.Cloud.Model
+{
+    public class TagItem
+    {
+        public IReadOnlyCollection<string> Tags { get; set; }
+        public DateTime CreatedTime { get; set; }
+        public string ImageDigest { get; set; }
+    }
+}

ServiceMonitor.Mailer/ServiceMonitor.Mailer.csproj

@@ -20,7 +20,6 @@
 
   <ItemGroup>
     <ProjectReference Include="..\ServiceMonitor.AWS\ServiceMonitor.AWS.csproj" />
-    <ProjectReference Include="..\ServiceMonitor.Cloud\ServiceMonitor.Cloud.csproj" />
   </ItemGroup>
 
   <ItemGroup>

ServiceMonitor.sln

@@ -21,6 +21,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceMonitor.Dashboard", "ServiceMonitor.Dashboard\ServiceMonitor.Dashboard.csproj", "{6B1DF568-BF56-45AA-9B70-D9F678B44A42}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceMonitor.Cleaner", "ServiceMonitor.Cleaner\ServiceMonitor.Cleaner.csproj", "{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -91,6 +93,18 @@ Global
 		{6B1DF568-BF56-45AA-9B70-D9F678B44A42}.Release|x64.Build.0 = Release|Any CPU
 		{6B1DF568-BF56-45AA-9B70-D9F678B44A42}.Release|x86.ActiveCfg = Release|Any CPU
 		{6B1DF568-BF56-45AA-9B70-D9F678B44A42}.Release|x86.Build.0 = Release|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x64.Build.0 = Debug|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x86.Build.0 = Debug|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x64.ActiveCfg = Release|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x64.Build.0 = Release|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x86.ActiveCfg = Release|Any CPU
+		{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE