Moving to S3 centralized storage.

ACM.Http/Collector.cs

@@ -7,19 +7,25 @@ namespace ACM.Http;
 
 public class Collector : ICollector {
  private readonly FileSystem fs_;
-
+ private readonly bool delta_;
+ private readonly bool forward_;
  public Collector(FileSystem fs) {
  fs_ = fs;
+ delta_ = Configuration.GetApcVar(ApcVariable.ACM_HTTP_DELTA) == "true";
+ forward_ = Configuration.GetApcVar(ApcVariable.ACM_HTTP_MODE) == "forward";
  }
 
  public async Task Consume(ConsumeContext<ArtifactCollectRequest> context) {
  string location = context.Message.location;
  string module = context.Message.module;
  string fp = fs_.GetArtifactPath(module, location);
- if ((!fs_.Exists(fp) && !fs_.Exists(fp + ".tmp")) || context.Message.force) {
- RemoteFile rf = new(location);
+ bool exists = await fs_.Exists(fp);
+ if (!exists || context.Message.force) {
+ RemoteFile rf = new(location, fs_);
  if (await rf.Get(fp)) {
- fs_.CreateDailyLink(module, location);
+ if (delta_) {
+ fs_.CreateDailyLink(module, location);
+ }
  }
  }
  }

ACM.Http/Program.cs

@@ -4,14 +4,33 @@
 using APC.Kernel.Constants;
 using APC.Kernel.Extensions;
 using APC.Kernel.Registrations;
+using Foundatio;
+using Foundatio.Storage;
 
 ModuleRegistration registration = new(ModuleType.ACM, typeof(Collector));
 registration.AddEndpoint("http");
 registration.AddEndpoint("https");
 
+/* SETUP STORAGE */
+MinioFileStorageConnectionStringBuilder connection =
+ new MinioFileStorageConnectionStringBuilder();
+
+connection.Region = Configuration.GetApcVar(ApcVariable.ACM_S3_REGION);
+connection.AccessKey = Configuration.GetApcVar(ApcVariable.ACM_S3_ACCESS_KEY);
+connection.SecretKey = Configuration.GetApcVar(ApcVariable.ACM_S3_SECRET_KEY);
+connection.EndPoint = Configuration.GetApcVar(ApcVariable.ACM_S3_ENDPOINT);
+connection.Bucket = Configuration.GetApcVar(ApcVariable.ACM_S3_BUCKET);
+
+MinioFileStorageOptions minio_options = new MinioFileStorageOptions() {
+ AutoCreateBucket = true,
+ ConnectionString = connection.ToString()
+};
+MinioFileStorage storage = new MinioFileStorage(minio_options);
+
 IHost host = Host.CreateDefaultBuilder(args)
  .ConfigureServices(services => {
  services.AddTelemetry(registration);
+ services.AddSingleton<IFileStorage>(storage);
  services.AddSingleton<FileSystem>();
  services.Register(registration);
  services.AddHostedService<Worker>();

ACM.Http/RemoteFile.cs

@@ -1,16 +1,19 @@
+using ACM.Kernel;
+
 namespace ACM.Http;
 
-internal class RemoteFile {
+public class RemoteFile {
  private const int BUFFER_SIZE_ = 8192;
  private static readonly HttpClient CLIENT_ = new();
  private readonly string url_;
+ private readonly FileSystem fs_;
 
- public RemoteFile(string url) {
+ public RemoteFile(string url, FileSystem fs) {
  url_ = url;
+ fs_ = fs;
  }
 
  public async Task<bool> Get(string filepath) {
- string tmp_file = filepath + ".tmp";
  HttpResponseMessage response =
  await CLIENT_.GetAsync(url_, HttpCompletionOption.ResponseHeadersRead);
  if (!response.IsSuccessStatusCode ||
@@ -19,68 +22,43 @@ public async Task<bool> Get(string filepath) {
  }
 
  long size = (long)response.Content.Headers.ContentLength;
- using Stream s = await response.Content.ReadAsStreamAsync();
+ await using Stream s = await response.Content.ReadAsStreamAsync();
  try {
- await ProcessStream(s, tmp_file);
+ await ProcessStream(s, filepath);
  } catch (Exception e) {
  s.Close();
- ClearFile(filepath);
- ClearFile(tmp_file);
+ await ClearFile(filepath);
  throw;
  }
 
  /* If downloaded file size matches remote, its complete */
- if (new FileInfo(tmp_file).Length == size) {
- /* Rename */
- File.Move(tmp_file, filepath);
- } else {
- File.Delete(tmp_file);
- return false;
+ if (await fs_.GetFileSize(filepath) == size) {
+ return true;
  }
 
- return true;
+ await ClearFile(filepath);
+ return false;
  }
 
- private static async Task
+ private async Task
  ProcessStream(Stream s, string filepath) {
- using FileStream fs = new(
- filepath,
- FileMode.Create,
- FileAccess.Write,
- FileShare.None,
- BUFFER_SIZE_,
- true
- );
- /* Progress */
- int total = 0;
- int read = -1;
- byte[] buffer = new byte[BUFFER_SIZE_];
+
  try {
- while (read != 0) {
- read = await s.ReadAsync(buffer, 0, buffer.Length);
- total += read;
- if (read == 0) {
- break;
- }
-
- await fs.WriteAsync(buffer, 0, read);
- }
+ await fs_.PutFile(filepath, s); 
  } catch (Exception e) {
- fs.Close();
- ClearFile(filepath);
+ await ClearFile(filepath);
  throw;
  }
-
- fs.Close();
  }
 
- private static bool ClearFile(string file) {
- if (!File.Exists(file)) {
+ private async Task<bool> ClearFile(string file) {
+
+ if (!(await fs_.Exists(file))) {
  return false;
  }
 
  Console.WriteLine($"Clearing {file}");
- File.Delete(file);
+ await fs_.Delete(file);
  return true;
  }
 }

ACM.Kernel/ACM.Kernel.csproj

@@ -10,4 +10,8 @@
  <ProjectReference Include="..\APC.Kernel\APC.Kernel.csproj"/>
  </ItemGroup>
 
+ <ItemGroup>
+ <PackageReference Include="Foundatio.Minio" Version="10.6.1" />
+ </ItemGroup>
+
 </Project>

ACM.Kernel/FileSystem.cs

@@ -1,15 +1,39 @@
 using System.Text.RegularExpressions;
 using APC.Kernel;
+using Foundatio.Storage;
 
 namespace ACM.Kernel;
 
 public class FileSystem {
- private static readonly string BASE_DIR_ =
+ private readonly string BASE_DIR_ =
  Configuration.GetApcVar(ApcVariable.APC_ACM_DIR);
 
- static FileSystem() {
+ private readonly IFileStorage storage_backend_;
+
+ public FileSystem(IFileStorage storage_backend) {
  Console.WriteLine($"Storage Directory: {BASE_DIR_}");
+ storage_backend_ = storage_backend;
+ }
+
+ public async Task<bool> Exists(string path) {
+ return await storage_backend_.ExistsAsync(path);
+ }
+
+ public async Task<bool> Delete(string path) {
+ return await storage_backend_.DeleteFileAsync(path);
+ }
+
+ public async Task<bool> Rename(string a, string b) { 
+ return await storage_backend_.RenameFileAsync(a, b);
+ }
+ public async Task<Stream> GetStream(string path) {
+ return await storage_backend_.GetFileStreamAsync(path);
+ }
+
+ public async Task<bool> PutFile(string path, Stream stream) {
+ return await storage_backend_.SaveFileAsync(path, stream);
  }
+
 
  private string GetDailyDeposit(string module) {
  string daily_deposit = Path.Join(BASE_DIR_, "Daily", module);
@@ -28,15 +52,13 @@ public void CreateDailyLink(string module, string uri_str) {
  public string GetArtifactPath(string module, string uri_str) {
  Uri uri = new(uri_str);
  string location = GetDiskLocation(uri);
- CreateModuleDirectory(module);
  string path = GetModulePath(module, location);
- CreateArtifactDirectory(Path.GetDirectoryName(path));
  return path;
  }
 
- public long GetFileSize(string filepath) {
- FileInfo info = new FileInfo(filepath);
- return info.Length;
+ public async Task<long> GetFileSize(string filepath) {
+ FileSpec spec = await storage_backend_.GetFileInfoAsync(filepath);
+ return spec.Size;
  }
 
  private string GetDiskLocation(Uri uri) {
@@ -48,18 +70,9 @@ private string CleanFilepath(string location) {
  }
 
  private string GetModulePath(string module, string filepath) {
- return Path.Join(GetModuleDir(module), filepath);
+ return Path.Join(module, filepath);
  }
-
- private void CreateArtifactDirectory(string path) {
- Directory.CreateDirectory(path);
- }
-
- private void CreateModuleDirectory(string module) {
- Directory.CreateDirectory(GetModuleDir(module));
- }
-
-
+
  public string GetModuleDir(string module, bool create = false) {
  string dir = Path.Join(BASE_DIR_, module);
  if (create) {
@@ -68,8 +81,4 @@ public string GetModuleDir(string module, bool create = false) {
 
  return dir;
  }
-
- public bool Exists(string filepath) {
- return File.Exists(filepath);
- }
 }

APC.Kernel/ApcVariable.cs

@@ -16,5 +16,13 @@ public enum ApcVariable {
  APC_ACM_DIR,
 
  APC_MONGO_STR,
- ACM_DAILY_MODE
+ ACM_HTTP_DELTA,
+ ACM_HTTP_MODE,
+
+ // S3
+ ACM_S3_ACCESS_KEY,
+ ACM_S3_SECRET_KEY,
+ ACM_S3_REGION,
+ ACM_S3_ENDPOINT,
+ ACM_S3_BUCKET
 }

APC.Kernel/Configuration.cs

@@ -25,7 +25,10 @@ public static class Configuration {
  ApcVariable.APC_OTEL_HOST, "http://localhost:4318"
  },
  {
- ApcVariable.ACM_DAILY_MODE, "link" // disabled, link, copy
+ ApcVariable.ACM_HTTP_DELTA, "true" // Create daily deltas
+ },
+ {
+ ApcVariable.ACM_HTTP_MODE, "lake" // lake, forward 
  }
  };
 

APC.Tester/APC.Tester.csproj

@@ -10,6 +10,7 @@
 
  <ItemGroup>
  <ProjectReference Include="..\ACM.Git\ACM.Git.csproj"/>
+ <ProjectReference Include="..\ACM.Http\ACM.Http.csproj" />
  <ProjectReference Include="..\APC.Infrastructure\APC.Infrastructure.csproj"/>
  <ProjectReference Include="..\APC.Skopeo\APC.Skopeo.csproj"/>
  <ProjectReference Include="..\APM.Helm\APM.Helm.csproj"/>

APC.Tester/Program.cs

@@ -1,24 +1,34 @@
 // See https://aka.ms/new-console-template for more information
 
+using ACM.Http;
+using ACM.Kernel;
 using APC.Kernel.Models;
 using APC.Skopeo;
 using APM.Jetbrains.IDE;
 using APM.Pypi;
+using Foundatio;
+using Foundatio.Storage;
 using IJetbrains = APM.Jetbrains.IDE.IJetbrains;
-
-Pypi pypi = new();
-SkopeoClient s = new();
-
-Artifact artifact = new() {
- id = "IIU",
- processor = "jetbrains-ide"
+MinioFileStorageConnectionStringBuilder connection =
+ new MinioFileStorageConnectionStringBuilder();
+connection.Region = "bleh";
+connection.AccessKey = "minio-apc";
+connection.SecretKey = "minio-apc";
+connection.EndPoint = "localhost:9000";
+connection.Bucket = "APC";
+
+MinioFileStorageOptions minio_options = new MinioFileStorageOptions() {
+ AutoCreateBucket = true,
+ ConnectionString = connection.ToString()
 };
-
-IJetbrains jetbrains = new Jetbrains();
-
-Artifact result = await jetbrains.ProcessArtifact(artifact);
+IFileStorage storage = new MinioFileStorage(minio_options);
+FileSystem fs = new FileSystem(storage);
 
 
+string url = "http://localhost:9001/static/js/main.379b5c5e.js";
+RemoteFile file = new RemoteFile(url, fs);
+string output = fs.GetArtifactPath("npm", url);
 
+await file.Get(output);
 //Artifact p = await pypi.ProcessArtifact(artifact);
 Console.WriteLine("---");

Compose/docker-compose.yml

@@ -1,6 +1,35 @@
 version: "3.8"
 services:
 # Infrastructure
+ # required to run container syncs
+ apc-minio:
+ image: minio/minio
+ restart: always
+ ports:
+ - "9000:9000"
+ - "9001:9001"
+ volumes:
+ - "./volumes/minio:/data"
+ environment:
+ MINIO_ROOT_USER: minio-dev
+ MINIO_ROOT_PASSWORD: minio-dev
+ command: server --console-address ":9001" /data
+
+ apc-internal-registry:
+ image: registry:2.8
+ restart: always
+ ports:
+ - "6000:5000"
+ environment:
+ REGISTRY_STORAGE: s3
+ REGISTRY_STORAGE_S3_ACCESSKEY: minio-apc
+ REGISTRY_STORAGE_S3_SECRETKEY: minio-apc
+ REGISTRY_STORAGE_S3_REGIONENDPOINT: http://apc-minio:9000
+ REGISTRY_STORAGE_S3_BUCKET: docker-registry
+ REGISTRY_STORAGE_S3_SECURE: "false"
+ REGISTRY_STORAGE_S3_region: ignored-for-minio
+ REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+
  apc-mongo:
  image: mongo
  restart: always