Release 0.20.0. See release notes or expand full commit message for d…

…etails.

[Improvements]
Upgraded IpfsShipyard.Ipfs.Http.Client to 0.6.0 with numerous notable improvements and breaking fixes.
Added KuboBootstrapper.EnableFilestore flag for use in tandem with the `NoCopy` option now available in the underlying API.
GetCidAsync supports the new `NoCopy` and all other AddFileOptions in the underlying API.
ContentAddressedSystemFolder supports the new `NoCopy` and automatically handles absolute paths for the filestore to use.
GetCidAsync now uses the new FilesystemApi.AddAsync method as the fallback implementation to crawl and upload an entire directory structure for any OwlCore.Storage implementation.

[Breaking]
See inherited breaking changes in IpfsShipyard.Ipfs.Core 0.7.0 and IpfsShipyard.Ipfs.Http.Client 0.6.0 release notes.

src/AesPasswordEncryptedPubSub.cs

@@ -111,7 +111,7 @@ public Task<IEnumerable<string>> SubscribedTopicsAsync(CancellationToken cancel
             var outputBytes = outputStream.ToBytes();
 
             outputStream.Seek(0, SeekOrigin.Begin);
-            return new PublishedMessage(publishedMessage.Sender, publishedMessage.Topics, publishedMessage.SequenceNumber, outputBytes, outputStream, publishedMessage.Size);
+            return new PublishedMessage(publishedMessage.Sender, publishedMessage.Topics, publishedMessage.SequenceNumber, outputBytes);
         }
         catch
         {

src/Cache/CachedCoreApi.cs

@@ -65,9 +65,6 @@ public CachedCoreApi(IModifiableFolder cacheFolder, ICoreApi inner)
     /// <inheritdoc/>
     public INameApi Name { get; }
 
-    /// <inheritdoc/>
-    public IObjectApi Object => Inner.Object;
-
     /// <inheritdoc/>
     public IPinApi Pin => Inner.Pin;
 

src/Extensions/GenericKuboExtensions.cs

@@ -26,7 +26,7 @@ public static partial class GenericKuboExtensions
             cid = Cid.Decode(ipnsResResult.Replace("/ipfs/", ""));
         }
 
-        var res = await client.Dag.GetAsync<TResult>(cid, cancellationToken);
+        var res = await client.Dag.GetAsync<TResult>(cid, cancel: cancellationToken);
 
         Guard.IsNotNull(res);
         return (res, cid);
@@ -49,7 +49,7 @@ public static partial class GenericKuboExtensions
             cid = Cid.Decode(ipnsResResult.Replace("/ipfs/", ""));
         }
 
-        var res = await client.Dag.GetAsync<TResult>(cid, cancellationToken);
+        var res = await client.Dag.GetAsync<TResult>(cid, cancel: cancellationToken);
 
         return (res, cid);
     }

src/Extensions/IpnsDagExtensions.cs

@@ -37,7 +37,7 @@ public static async Task TransformIpnsDagAsync<TTransformType>(this ICoreApi cli
 
         // Resolve data
         progress?.Report(IpnsUpdateState.ResolvingDag);
-        var data = await client.Dag.GetAsync<TTransformType>(cid, cancellationToken);
+        var data = await client.Dag.GetAsync<TTransformType>(cid, cancel: cancellationToken);
         if (data is null)
             throw new InvalidOperationException("Failed to resolve data from the provided CID.");
 

src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs

@@ -1,4 +1,5 @@
-using CommunityToolkit.Diagnostics;
+using System.Diagnostics;
+using CommunityToolkit.Diagnostics;
 using Ipfs;
 using Ipfs.CoreApi;
 using OwlCore.Kubo;
@@ -29,9 +30,42 @@ public ContentAddressedSystemFolder(string path, ICoreApi client)
     /// <inheritdoc/>
     public async Task<Cid> GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken)
     {
-        var res = await Client.FileSystem.AddDirectoryAsync(Id, recursive: true, addFileOptions, cancellationToken);
+        FilePart[] fileParts = [];
+        FolderPart[] folderParts = [new FolderPart { Name = Name }];
 
-        Guard.IsTrue(res.IsDirectory);
-        return res.ToLink().Id;
+        // Get child file and folder parts recursively
+        var q = new Queue<SystemFolder>([this]);
+        while (q.Count > 0)
+        {
+            await foreach (var item in q.Dequeue().GetItemsAsync(StorableType.All, cancellationToken))
+            {
+                var relativePath = await this.GetRelativePathToAsync(item, cancellationToken);
+                var adjustedRelativePath = $"{Name}{relativePath}";
+
+                if (item is SystemFile file)
+                {
+                    fileParts = [.. fileParts, new FilePart { Name = adjustedRelativePath, Data = new FileStream(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous), AbsolutePath = file.Path }];
+                }
+                else if (item is SystemFolder folder)
+                {
+                    folderParts = [.. folderParts, new FolderPart { Name = adjustedRelativePath }];
+                    q.Enqueue(folder);
+                }
+            }
+        }
+
+        Cid? rootCid = null;
+
+        await foreach (var node in Client.FileSystem.AddAsync(fileParts, folderParts, addFileOptions, cancellationToken))
+        {
+            var filePart = fileParts.FirstOrDefault(x => x.Name == node.Name);
+            filePart?.Data?.Dispose();
+
+            if (node.Name == Name)
+                rootCid = node.Id;
+        }
+
+        Guard.IsNotNull(rootCid);
+        return rootCid;
     }
 }

src/Extensions/StorableKuboExtensions.cs

@@ -60,6 +60,47 @@ public static async Task<Cid> GetCidAsync(this IStorable item, ICoreApi client,
             return res.ToLink().Id;
         }
 
-        throw new NotSupportedException($"Provided storable item is not a supported type. Provide an {nameof(IFile)} or an implementation of {nameof(IGetCid)}");
+        // Process as folder
+        if (item is not IFolder folder)
+            throw new NotSupportedException($"Unsupported implementation of {nameof(IStorable)}: {item.GetType().Name}");
+
+        // Get child file and folder parts recursively
+        FilePart[] fileParts = [];
+        FolderPart[] folderParts = [new FolderPart { Name = folder.Name }];
+
+        var q = new Queue<IFolder>([folder]);
+        while (q.Count > 0)
+        {
+            await foreach (var folderItem in q.Dequeue().GetItemsAsync(StorableType.All, cancellationToken))
+            {
+                var relativePath = await folder.GetRelativePathToAsync(folderItem, cancellationToken);
+                var adjustedRelativePath = $"{folder.Name}{relativePath}";
+
+                if (folderItem is IFile innerFile)
+                {
+                    fileParts = [.. fileParts, new FilePart { Name = adjustedRelativePath, Data = await innerFile.OpenReadAsync(cancellationToken) }];
+                }
+                else if (folderItem is IFolder innerFolder)
+                {
+                    folderParts = [.. folderParts, new FolderPart { Name = adjustedRelativePath }];
+                    q.Enqueue(innerFolder);
+                }
+            }
+        }
+
+        Cid? rootCid = null;
+
+        // Add to ipfs, dispose of data streams as they're processed
+        await foreach (var node in client.FileSystem.AddAsync(fileParts, folderParts, addFileOptions, cancellationToken))
+        {
+            var filePart = fileParts.FirstOrDefault(x => x.Name == node.Name);
+            filePart?.Data?.Dispose();
+
+            if (node.Name == folder.Name)
+                rootCid = node.Id;
+        }
+
+        Guard.IsNotNull(rootCid);
+        return rootCid;
     }
 }

src/KuboBootstrapper.cs

@@ -1,12 +1,12 @@
 using CommunityToolkit.Diagnostics;
 using Ipfs;
+using Ipfs.CoreApi;
 using Ipfs.Http;
 using Newtonsoft.Json.Linq;
 using OwlCore.Extensions;
 using OwlCore.Storage;
 using OwlCore.Storage.System.IO;
 using System.Diagnostics;
-using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
 
@@ -151,6 +151,14 @@ public KuboBootstrapper(string repoPath)
     /// </remarks>
     public bool UseAcceleratedDHTClient { get; set; }
 
+    /// <summary>
+    /// Gets or sets a value indicating whether the filestore feature should be enabled. Allows files to be added without duplicating the space they take up on disk.
+    /// </summary>
+    /// <remarks>
+    /// To add files using the filestore, pass the NoCopy option to <see cref="AddFileOptions"/> in the <see cref="IFileSystemApi.AddAsync(FilePart[], FolderPart[], AddFileOptions?, CancellationToken)"/> method.
+    /// </remarks>
+    public bool EnableFilestore { get; set; }
+
     /// <summary>
     /// The Kubo profiles that will be applied before starting the daemon.
     /// </summary>
@@ -474,6 +482,7 @@ public virtual async Task ApplySettingsAsync(CancellationToken cancellationToken
             // ignored
         }
 
+        await ApplyExperimentalConfigSettingsAsync(cancellationToken);
         await ApplyRoutingSettingsAsync(cancellationToken);
         await ApplyPortSettingsAsync(cancellationToken);
         await ApplyStartupProfileSettingsAsync(cancellationToken);
@@ -490,6 +499,7 @@ protected virtual Task ApplyStartupProfileSettingsAsync(CancellationToken cancel
         // Startup profiles
         foreach (var profile in StartupProfiles)
             RunExecutable(_kuboBinaryFile, $"config --repo-dir \"{RepoPath}\" profile apply {profile}", throwOnError: true);
+
         return Task.CompletedTask;
     }
 
@@ -523,6 +533,20 @@ protected virtual async Task ApplyPortSettingsAsync(CancellationToken cancellati
         }
     }
 
+    /// <summary>
+    /// Initializes the local node with the provided experimental settings.
+    /// </summary>
+    /// <param name="cancellationToken">A token that can be used to cancel the ongoing operation.</param>
+    /// <returns>A <see cref="Task"/> that represents the asynchronous operation.</returns>
+    protected virtual Task ApplyExperimentalConfigSettingsAsync(CancellationToken cancellationToken)
+    {
+        Guard.IsNotNullOrWhiteSpace(_kuboBinaryFile?.Path);
+
+        RunExecutable(_kuboBinaryFile, $"config Experimental.FilestoreEnabled \"{EnableFilestore.ToString().ToLower()}\" --json --repo-dir \"{RepoPath}\"", throwOnError: true);
+
+        return Task.CompletedTask;
+    }
+
     /// <summary>
     /// Initializes the local node with the provided routing settings.
     /// </summary>
@@ -532,8 +556,8 @@ protected virtual Task ApplyRoutingSettingsAsync(CancellationToken cancellationT
         Guard.IsNotNullOrWhiteSpace(_kuboBinaryFile?.Path);
 
         RunExecutable(_kuboBinaryFile, $"config Routing.Type {RoutingMode.ToString().ToLowerInvariant()} --repo-dir \"{RepoPath}\"", throwOnError: true);
-
         RunExecutable(_kuboBinaryFile, $"config Routing.AcceleratedDHTClient \"{UseAcceleratedDHTClient.ToString().ToLower()}\" --json --repo-dir \"{RepoPath}\"", throwOnError: true);
+
         return Task.CompletedTask;
     }
 

src/Models/PublishedMessage.cs

@@ -11,14 +11,12 @@ public class PublishedMessage : IPublishedMessage
     /// <summary>
     /// Creates a new instance of <see cref="PublishedMessage"/>.
     /// </summary>
-    public PublishedMessage(Peer sender, IEnumerable<string> topics, byte[] sequenceNumber, byte[] dataBytes, Stream dataStream, long size)
+    public PublishedMessage(Peer sender, IEnumerable<string> topics, byte[] sequenceNumber, byte[] dataBytes)
     {
         Sender = sender;
         Topics = topics;
         SequenceNumber = sequenceNumber;
         DataBytes = dataBytes;
-        DataStream = dataStream;
-        Size = size;
     }
 
     /// <inheritdoc/>
@@ -32,13 +30,4 @@ public PublishedMessage(Peer sender, IEnumerable<string> topics, byte[] sequence
 
     /// <inheritdoc/>
     public byte[] DataBytes { get; }
-
-    /// <inheritdoc/>
-    public Stream DataStream { get; }
-
-    /// <inheritdoc/>
-    public Cid Id { get; } = MultiHash.ComputeHash(Encoding.UTF8.GetBytes(nameof(PublishedMessage)));
-
-    /// <inheritdoc/>
-    public long Size { get; }
 }

src/OwlCore.Kubo.csproj

@@ -14,13 +14,24 @@
     <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
 
     <Author>Arlo Godfrey</Author>
-    <Version>0.19.1</Version>
+    <Version>0.20.0</Version>
     <Product>OwlCore</Product>
     <Description>
       An essential toolkit for Kubo, IPFS and the distributed web. 
     </Description>
     <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
     <PackageReleaseNotes>
+--- 0.20.0 ---
+[Improvements]
+Upgraded IpfsShipyard.Ipfs.Http.Client to 0.6.0 with numerous notable improvements and breaking fixes.
+Added KuboBootstrapper.EnableFilestore flag for use in tandem with the `NoCopy` option now available in the underlying API.
+GetCidAsync supports the new `NoCopy` and all other AddFileOptions in the underlying API.
+ContentAddressedSystemFolder supports the new `NoCopy` and automatically handles absolute paths for the filestore to use.
+GetCidAsync now uses the new FilesystemApi.AddAsync method as the fallback implementation to crawl and upload an entire directory structure for any OwlCore.Storage implementation. 
+
+[Breaking]
+See inherited breaking changes in IpfsShipyard.Ipfs.Core 0.7.0 and IpfsShipyard.Ipfs.Http.Client 0.6.0 release notes. 
+
 --- 0.19.1 ---
 [Improvements]
 Updated vulnerable System.Text.Json to latest stable 9.0.0.
@@ -468,7 +479,7 @@ Added unit tests.
   <ItemGroup>
     <PackageReference Include="CommunityToolkit.Common" Version="8.3.2" />
     <PackageReference Include="CommunityToolkit.Diagnostics" Version="8.3.2" />
-    <PackageReference Include="IpfsShipyard.Ipfs.Http.Client" Version="0.5.1" />
+    <PackageReference Include="IpfsShipyard.Ipfs.Http.Client" Version="0.6.0" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.0" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
     <PackageReference Include="OwlCore.ComponentModel.Settings" Version="0.1.1" />

tests/OwlCore.Kubo.Tests/Extensions/StorageExtensions.cs

@@ -11,11 +11,9 @@ public static async IAsyncEnumerable<IChildFile> CreateFilesAsync(this IModifiab
             yield return await folder.CreateFileAsync(getFileName(i), overwrite: true, cancellationToken: cancellationToken);
     }
 
-    public static async Task WriteRandomBytes(this IFile file, long numberOfBytes, int bufferSize, CancellationToken cancellationToken)
+    public static async Task WriteRandomBytesAsync(this IFile file, long numberOfBytes, int bufferSize, CancellationToken cancellationToken)
     {
-        var rnd = new Random();
-
-        await using var fileStream = await file.OpenWriteAsync(cancellationToken);
+        var fileStream = await file.OpenWriteAsync(cancellationToken);
 
         var bytes = new byte[bufferSize];
         var bytesWritten = 0L;
@@ -30,12 +28,14 @@ public static async Task WriteRandomBytes(this IFile file, long numberOfBytes, i
 
             if (bytes.Length != bufferSize)
                 bytes = new byte[bufferSize];
-            
-            rnd.NextBytes(bytes);
+
+            Random.Shared.NextBytes(bytes);
 
             await fileStream.WriteAsync(bytes, cancellationToken);
             bytesWritten += bufferSize;
         }
+
+        await fileStream.DisposeAsync();
     }
 
     public static async Task AssertStreamEqualAsync(this Stream srcFileStream, Stream destFileStream, int bufferSize, CancellationToken cancellationToken)

tests/OwlCore.Kubo.Tests/FilestoreTests.cs

@@ -0,0 +1,51 @@
+using OwlCore.Storage.System.IO;
+using System.Diagnostics;
+using System.Text;
+using CommunityToolkit.Diagnostics;
+using OwlCore.Storage;
+
+namespace OwlCore.Kubo.Tests
+{
+    [TestClass]
+    public class FilestoreTests
+    {
+        [TestMethod]
+        public async Task TestFilestoreAsync()
+        {
+            Guard.IsNotNull(TestFixture.Bootstrapper);
+            var client = TestFixture.Client;
+
+            var workingFolder = (IModifiableFolder?)await TestFixture.Bootstrapper.RepoFolder.GetParentAsync();
+            Guard.IsNotNull(workingFolder);
+
+            var testFolder = (SystemFolder)await workingFolder.CreateFolderAsync("in", overwrite: true);
+
+            // Create random data in working folder
+            await foreach (var file in testFolder.CreateFilesAsync(2, i => $"{i}.bin", CancellationToken.None))
+                await file.WriteRandomBytesAsync(512, 512, CancellationToken.None);
+
+            // Add the same data to filestore
+            var filestoreFolderCid = await testFolder.GetCidAsync(client, new Ipfs.CoreApi.AddFileOptions { NoCopy = true, CidVersion = 1, }, default);
+
+            // List the filestore
+            var items = await client.Filestore.ListAsync().ToListAsync();
+            Assert.AreEqual(2, items.Count);
+
+            // Verify duplicate don't exist yet
+            var duplicates = await client.Filestore.DupsAsync().ToListAsync();
+            Assert.AreEqual(0, duplicates.Count);
+
+            // Add to ipfs (without filestore)
+            var folderCid = await testFolder.GetCidAsync(client, new Ipfs.CoreApi.AddFileOptions { NoCopy = false, RawLeaves = true, CidVersion = 1, }, default);
+            Assert.AreEqual(folderCid, filestoreFolderCid);
+
+            // Verify the filestore
+            items = await client.Filestore.VerifyObjectsAsync().ToListAsync();
+            Assert.AreEqual(2, items.Count);
+
+            // Verify duplicates now exist
+            duplicates = await client.Filestore.DupsAsync().ToListAsync();
+            Assert.AreEqual(2, duplicates.Count);
+        }
+    }
+}