diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 991abab..8ddc36d 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - run-cron: - name: "Run Cron" + run-report: + name: "Run Report" runs-on: ubuntu-22.04 permissions: id-token: write # This is required for requesting the JWT @@ -33,3 +33,27 @@ jobs: env: TO_EMAIL: ${{ secrets.TO_EMAIL }} FROM_EMAIL: ${{ secrets.FROM_EMAIL }} + run-cleaner: + name: "Run Cleaner" + runs-on: ubuntu-22.04 + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + steps: + - uses: actions/checkout@v3 + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6.0.x + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }} + aws-region: "ap-southeast-1" + - name: Restore dependencies + run: dotnet restore + - name: Build + run: dotnet build --no-restore + - name: Test + run: dotnet run --no-build --project ServiceMonitor.Cleaner \ No newline at end of file diff --git a/ServiceMonitor.AWS/ECR.cs b/ServiceMonitor.AWS/ECR.cs index 089d11b..871959b 100644 --- a/ServiceMonitor.AWS/ECR.cs +++ b/ServiceMonitor.AWS/ECR.cs @@ -8,16 +8,22 @@ using Amazon.ECR; using Amazon.ECR.Model; using System.Linq; +using ServiceMonitor.Cloud.Model; +using Microsoft.Extensions.Logging; namespace ServiceMonitor.AWS { public class ECR : IImage { private readonly IServiceProvider _serviceProvider; - public ECR(IServiceProvider serviceProvider) + private readonly ILogger _logger; + + public ECR(IServiceProvider serviceProvider, ILogger logger) { _serviceProvider = serviceProvider; + _logger = logger; } + public async Task> GetImagesAsync(string region, CancellationToken cancellationToken = default) { // TODO: need better solution for client resolution @@ -43,5 +49,91 @@ public async Task> GetImagesAsync(string region, Canc return list; } + + public async Task> GetTags(string region, string repoName, CancellationToken cancellationToken = default) + { + var ecrRegion = RegionEndpoint.GetBySystemName(region); + using var ecrClient = ActivatorUtilities.CreateInstance(_serviceProvider, ecrRegion); + if (ecrClient == null) + { + throw new ArgumentNullException(nameof(ecrClient)); + } + var imageListRequest = new DescribeImagesRequest + { + RepositoryName = repoName, + Filter = new Amazon.ECR.Model.DescribeImagesFilter + { + TagStatus = TagStatus.TAGGED, + } + }; + var tagsResources = await ecrClient.DescribeImagesAsync(imageListRequest); + return tagsResources.ImageDetails.Select(x => new TagItem + { + CreatedTime = x.ImagePushedAt, + Tags = x.ImageTags, + ImageDigest = x.ImageDigest, + }).ToList(); + } + + public async Task<(List, List)> DeleteTags(string region, string repoName, ICollection tags, CancellationToken cancellationToken = default) + { + var sample = tags.FirstOrDefault(); + if (sample == null) + { + return (new List(), new List()); + } + + var ecrRegion = RegionEndpoint.GetBySystemName(region); + using var ecrClient = ActivatorUtilities.CreateInstance(_serviceProvider, ecrRegion); + if (ecrClient == null) + { + throw new ArgumentNullException(nameof(ecrClient)); + } + + var isShaMode = sample.Tags.Contains("sha-"); + var deleteImages = new List(); + if (isShaMode) + { + foreach (var tag in tags) + { + if (tag.Tags.Contains("main") || tag.Tags.Contains("latest") || tag.Tags.Contains("master")) + { + continue; + } + deleteImages.Add(new ImageIdentifier + { + ImageDigest = tag.ImageDigest, + }); + } + } + else + { + var sorted = tags.OrderByDescending(x => x.CreatedTime).Select(x => new ImageIdentifier + { + ImageDigest = x.ImageDigest, + }).ToList(); + sorted.RemoveAt(0); + deleteImages.AddRange(sorted); + } + + if (deleteImages.Count == 0) + { + return (new List(), new List()); + } + + _logger.LogInformation("Will remove {} tags", deleteImages.Count); + + var deleteImageRequest = new BatchDeleteImageRequest + { + RepositoryName = repoName, + ImageIds = deleteImages, + }; + + var batchResponse = await ecrClient.BatchDeleteImageAsync(deleteImageRequest); + + _logger.LogInformation("Success remove {} and fail {}", batchResponse.ImageIds.Count, batchResponse.Failures.Count); + + return (batchResponse.ImageIds.Select(x => x.ImageTag).ToList(), batchResponse.Failures.Select(x => x.ImageId.ImageTag).ToList()); + } } } diff --git a/ServiceMonitor.AWS/ServiceMonitor.AWS.csproj b/ServiceMonitor.AWS/ServiceMonitor.AWS.csproj index 99124da..c1650b8 100644 --- a/ServiceMonitor.AWS/ServiceMonitor.AWS.csproj +++ b/ServiceMonitor.AWS/ServiceMonitor.AWS.csproj @@ -11,6 +11,7 @@ + diff --git a/ServiceMonitor.Cleaner/Program.cs b/ServiceMonitor.Cleaner/Program.cs new file mode 100644 index 0000000..2395f90 --- /dev/null +++ b/ServiceMonitor.Cleaner/Program.cs @@ -0,0 +1,64 @@ + +using Amazon; +using Amazon.EC2.Model; +using Amazon.SimpleEmailV2; +using Amazon.SimpleEmailV2.Model; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceMonitor.AWS; +using ServiceMonitor.Cloud; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +IConfiguration config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", true) + .AddEnvironmentVariables() + .Build(); + +var serviceCollection = new ServiceCollection(); +serviceCollection.AddSingleton(config); +serviceCollection.AddLogging(configure => +{ + configure.ClearProviders(); + configure.AddConfiguration(config.GetSection("Logging")); + configure.AddConsole(); +}); +serviceCollection.AddAWSService(); +serviceCollection.AddScoped(); +serviceCollection.AddScoped(); +serviceCollection.AddScoped(); +serviceCollection.AddScoped(); +serviceCollection.AddScoped(); + +var serviceProvider = serviceCollection.BuildServiceProvider(); + +var logger = serviceProvider.GetRequiredService().CreateLogger(); +logger.LogDebug("Starting application"); + +var regions = new List +{ + "ap-southeast-1", + "ap-southeast-3", +}; + +var containerImages = serviceProvider.GetRequiredService(); +foreach (var region in regions) +{ + var images = await containerImages.GetImagesAsync(region); + foreach (var image in images) + { + var tags = await containerImages.GetTags(region, image.Name); + var (success, failure) = await containerImages.DeleteTags(region, image.Name, tags); + logger.LogInformation("{}:{}:{}", image.Name, JsonSerializer.Serialize(success, new JsonSerializerOptions + { + WriteIndented = true, + }), + JsonSerializer.Serialize(failure, new JsonSerializerOptions + { + WriteIndented = true, + })); + } +} diff --git a/ServiceMonitor.Cleaner/ServiceMonitor.Cleaner.csproj b/ServiceMonitor.Cleaner/ServiceMonitor.Cleaner.csproj new file mode 100644 index 0000000..16cbe15 --- /dev/null +++ b/ServiceMonitor.Cleaner/ServiceMonitor.Cleaner.csproj @@ -0,0 +1,25 @@ + + + + Exe + net6.0 + enable + enable + + + + + + + + + + + + + + + + + + diff --git a/ServiceMonitor.Cloud/IImage.cs b/ServiceMonitor.Cloud/IImage.cs index 6f5d657..4f15c4c 100644 --- a/ServiceMonitor.Cloud/IImage.cs +++ b/ServiceMonitor.Cloud/IImage.cs @@ -1,11 +1,14 @@ using System.Collections.Generic; using System.Threading.Tasks; using System.Threading; +using ServiceMonitor.Cloud.Model; namespace ServiceMonitor.Cloud { public interface IImage { Task> GetImagesAsync(string region, CancellationToken cancellationToken = default); + Task> GetTags(string region, string repoName, CancellationToken cancellationToken = default); + Task<(List, List)> DeleteTags(string region, string repoName, ICollection tags, CancellationToken cancellationToken = default); } } diff --git a/ServiceMonitor.Cloud/Model/TagItem.cs b/ServiceMonitor.Cloud/Model/TagItem.cs new file mode 100644 index 0000000..ab6c03e --- /dev/null +++ b/ServiceMonitor.Cloud/Model/TagItem.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Text; + +namespace ServiceMonitor.Cloud.Model +{ + public class TagItem + { + public IReadOnlyCollection Tags { get; set; } + public DateTime CreatedTime { get; set; } + public string ImageDigest { get; set; } + } +} diff --git a/ServiceMonitor.Mailer/ServiceMonitor.Mailer.csproj b/ServiceMonitor.Mailer/ServiceMonitor.Mailer.csproj index 14dcf83..af94086 100644 --- a/ServiceMonitor.Mailer/ServiceMonitor.Mailer.csproj +++ b/ServiceMonitor.Mailer/ServiceMonitor.Mailer.csproj @@ -20,7 +20,6 @@ - diff --git a/ServiceMonitor.sln b/ServiceMonitor.sln index 28b2262..0f7b377 100644 --- a/ServiceMonitor.sln +++ b/ServiceMonitor.sln @@ -21,6 +21,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceMonitor.Dashboard", "ServiceMonitor.Dashboard\ServiceMonitor.Dashboard.csproj", "{6B1DF568-BF56-45AA-9B70-D9F678B44A42}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceMonitor.Cleaner", "ServiceMonitor.Cleaner\ServiceMonitor.Cleaner.csproj", "{CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -91,6 +93,18 @@ Global {6B1DF568-BF56-45AA-9B70-D9F678B44A42}.Release|x64.Build.0 = Release|Any CPU {6B1DF568-BF56-45AA-9B70-D9F678B44A42}.Release|x86.ActiveCfg = Release|Any CPU {6B1DF568-BF56-45AA-9B70-D9F678B44A42}.Release|x86.Build.0 = Release|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x64.ActiveCfg = Debug|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x64.Build.0 = Debug|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x86.ActiveCfg = Debug|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Debug|x86.Build.0 = Debug|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|Any CPU.Build.0 = Release|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x64.ActiveCfg = Release|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x64.Build.0 = Release|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x86.ActiveCfg = Release|Any CPU + {CA2C1BED-B547-40CC-92BC-F9C63D17F6EA}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE