diff --git a/src/Extensions/IAddFileToGetCid.cs b/src/Extensions/IAddFileToGetCid.cs new file mode 100644 index 0000000..2948042 --- /dev/null +++ b/src/Extensions/IAddFileToGetCid.cs @@ -0,0 +1,19 @@ +using Ipfs; +using Ipfs.CoreApi; +using OwlCore.Storage; + +namespace OwlCore.Kubo; + +/// +/// Implementations are capable of providing a CID for the current content by adding it ipfs. +/// +public partial interface IAddFileToGetCid : IStorable +{ + /// + /// Gets the CID of the storable item. + /// + /// The add file options to use when computing the cid for this storable. + /// A token that can be used to cancel the ongoing operation. + /// + public Task GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken); +} diff --git a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs index 12d20e4..bbb4295 100644 --- a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs +++ b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFile.cs @@ -8,7 +8,7 @@ namespace OwlCore.Storage.System.IO; /// /// An implementation of with added support for . /// -public class ContentAddressedSystemFile : SystemFile, IGetCid +public class ContentAddressedSystemFile : SystemFile, IAddFileToGetCid { /// /// Creates a new instance of . @@ -27,13 +27,9 @@ public ContentAddressedSystemFile(string path, ICoreApi client) public ICoreApi Client { get; } /// - public async Task GetCidAsync(CancellationToken cancellationToken) + public async Task GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken) { - var res = await Client.FileSystem.AddFileAsync(Id, new() - { - OnlyHash = true, - Pin = false - }, cancellationToken); + var res = await Client.FileSystem.AddFileAsync(Id, addFileOptions, cancellationToken); Guard.IsFalse(res.IsDirectory); return res.ToLink().Id; diff --git a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs index 21d1d0c..c8f151b 100644 --- a/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs +++ b/src/Extensions/OwlCore.Storage.System.IO/ContentAddressedSystemFolder.cs @@ -8,7 +8,7 @@ namespace OwlCore.Storage.System.IO; /// /// An implementation of with added support for . /// -public class ContentAddressedSystemFolder : SystemFolder, IGetCid +public class ContentAddressedSystemFolder : SystemFolder, IAddFileToGetCid { /// /// Creates a new instance of . @@ -27,13 +27,9 @@ public ContentAddressedSystemFolder(string path, ICoreApi client) public ICoreApi Client { get; } /// - public async Task GetCidAsync(CancellationToken cancellationToken) + public async Task GetCidAsync(AddFileOptions addFileOptions, CancellationToken cancellationToken) { - var res = await Client.FileSystem.AddDirectoryAsync(Id, recursive: true, new() - { - OnlyHash = true, - Pin = false, - }, cancellationToken); + var res = await Client.FileSystem.AddDirectoryAsync(Id, recursive: true, addFileOptions, cancellationToken); Guard.IsTrue(res.IsDirectory); return res.ToLink().Id; diff --git a/src/Extensions/StorableKuboExtensions.cs b/src/Extensions/StorableKuboExtensions.cs index f9fa842..c1b3662 100644 --- a/src/Extensions/StorableKuboExtensions.cs +++ b/src/Extensions/StorableKuboExtensions.cs @@ -12,8 +12,16 @@ namespace OwlCore.Kubo; /// public static partial class StorableKuboExtensions { - /// - public static async Task GetCidAsync(this IStorable item, ICoreApi client, CancellationToken cancellationToken) + /// + /// Gets a CID for the provided . If possible, a CID will be provided without adding the item to ipfs, otherwise the will be used to add content to ipfs and compute the cid. + /// + /// The storable to get the cid for. + /// The client to use for communicating with ipfs. + /// The options to use when adding content from the to ipfs. + /// A token that can be used to cancel the ongoing operation. + /// A task containing the cid of the . + /// An unsupported implementation of was provided for . + public static async Task GetCidAsync(this IStorable item, ICoreApi client, AddFileOptions addFileOptions, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -24,15 +32,21 @@ public static async Task GetCidAsync(this IStorable item, ICoreApi client, // You'd typically only do this if you can interact your Kubo node faster than a manually piped stream, // or if your implementation of IGetCid has enhanced capabilities (e.g. folder support). // --- - if (item is not IGetCid && item is SystemFile systemFile) + + // Get cid without adding to ipfs, if possible + if (item is IGetCid getCid) + return await getCid.GetCidAsync(cancellationToken); + + // Get cid by adding content to ipfs. + if (item is not IAddFileToGetCid && item is SystemFile systemFile) item = new ContentAddressedSystemFile(systemFile.Path, client); - if (item is not IGetCid && item is SystemFolder systemFolder) + if (item is not IAddFileToGetCid && item is SystemFolder systemFolder) item = new ContentAddressedSystemFolder(systemFolder.Path, client); // If the implementation can handle content addressing directly, use that. - if (item is IGetCid contentAddressedStorable) - return await contentAddressedStorable.GetCidAsync(cancellationToken); + if (item is IAddFileToGetCid contentAddressedStorable) + return await contentAddressedStorable.GetCidAsync(addFileOptions, cancellationToken); // Otherwise, a fallback approach that manually connects the streams together. // The Kubo API doesn't support this scenario for folders, without assuming that the Id is a local path, @@ -40,12 +54,7 @@ public static async Task GetCidAsync(this IStorable item, ICoreApi client, if (item is IFile file) { using var stream = await file.OpenStreamAsync(FileAccess.Read, cancellationToken); - - var res = await client.FileSystem.AddAsync(stream, file.Name, new() - { - OnlyHash = true, - Pin = false, - }, cancellationToken); + var res = await client.FileSystem.AddAsync(stream, file.Name, addFileOptions, cancellationToken); Guard.IsFalse(res.IsDirectory); return res.ToLink().Id; diff --git a/src/IpnsFolder.cs b/src/IpnsFolder.cs index 31c0f88..482c550 100644 --- a/src/IpnsFolder.cs +++ b/src/IpnsFolder.cs @@ -71,7 +71,7 @@ public IpnsFolder(string ipnsAddress, ICoreApi client) /// public virtual async IAsyncEnumerable GetItemsAsync(StorableType type = StorableType.All, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var cid = await GetCidAsync(cancellationToken); + var cid = await GetCidAsync(Id, cancellationToken); var itemInfo = await Client.FileSystem.ListAsync(cid, cancellationToken); Guard.IsTrue(itemInfo.IsDirectory); diff --git a/src/MfsFolder.Modifiable.cs b/src/MfsFolder.Modifiable.cs index acd0bd7..e9c1e74 100644 --- a/src/MfsFolder.Modifiable.cs +++ b/src/MfsFolder.Modifiable.cs @@ -1,4 +1,5 @@ using CommunityToolkit.Diagnostics; +using Ipfs.CoreApi; using OwlCore.Kubo.FolderWatchers; using OwlCore.Storage; @@ -11,6 +12,11 @@ public partial class MfsFolder : IModifiableFolder, IMoveFrom, ICreateCopyOf /// public TimeSpan UpdateCheckInterval { get; } = TimeSpan.FromSeconds(10); + /// + /// The options to use when adding content to this folder on ipfs. + /// + public AddFileOptions AddFileOptions { get; set; } = new(); + /// public virtual async Task DeleteAsync(IStorableChild item, CancellationToken cancellationToken = default) { @@ -21,24 +27,40 @@ public virtual async Task DeleteAsync(IStorableChild item, CancellationToken can } /// - public Task CreateCopyOfAsync(IFile fileToCopy, bool overwrite, CancellationToken cancellationToken, - CreateCopyOfDelegate fallback) + public async Task CreateCopyOfAsync(IFile fileToCopy, bool overwrite, CancellationToken cancellationToken, CreateCopyOfDelegate fallback) { if (fileToCopy is MfsFile mfsFile) - return CreateCopyOfAsync(mfsFile, overwrite, cancellationToken); + return await CreateCopyOfAsync(mfsFile, overwrite, cancellationToken); if (fileToCopy is IpfsFile ipfsFile) - return CreateCopyOfAsync(ipfsFile, overwrite, cancellationToken); + return await CreateCopyOfAsync(ipfsFile, overwrite, cancellationToken); if (fileToCopy is IpnsFile ipnsFile) - return CreateCopyOfAsync(ipnsFile, overwrite, cancellationToken); + return await CreateCopyOfAsync(ipnsFile, overwrite, cancellationToken); + + if (fileToCopy is IGetCid getCid) + { + var cid = await getCid.GetCidAsync(cancellationToken); + + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync($"/ipfs/{cid}", newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); + } - return fallback(this, fileToCopy, overwrite, cancellationToken); + if (fileToCopy is IAddFileToGetCid addFileToGetCid) + { + var cid = await addFileToGetCid.GetCidAsync(AddFileOptions, cancellationToken); + + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync($"/ipfs/{cid}", newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); + } + + return await fallback(this, fileToCopy, overwrite, cancellationToken); } /// - public Task MoveFromAsync(IChildFile fileToMove, IModifiableFolder source, bool overwrite, CancellationToken cancellationToken, - MoveFromDelegate fallback) + public Task MoveFromAsync(IChildFile fileToMove, IModifiableFolder source, bool overwrite, CancellationToken cancellationToken, MoveFromDelegate fallback) { if (fileToMove is MfsFile mfsFile) return MoveFromAsync(mfsFile, source, overwrite, cancellationToken); @@ -51,8 +73,9 @@ public virtual async Task CreateCopyOfAsync(MfsFile fileToCopy, bool { cancellationToken.ThrowIfCancellationRequested(); - await Client.Mfs.CopyAsync(fileToCopy.Path, Path, cancel: cancellationToken); - return new MfsFile($"{Path}{fileToCopy.Name}", Client); + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync(fileToCopy.Path, newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); } /// @@ -60,8 +83,9 @@ public virtual async Task CreateCopyOfAsync(IpfsFile fileToCopy, boo { cancellationToken.ThrowIfCancellationRequested(); - await Client.Mfs.CopyAsync($"/ipfs/{fileToCopy.Id}", Path, cancel: cancellationToken); - return new MfsFile($"{Path}{fileToCopy.Name}", Client); + var newPath = $"{Path}{fileToCopy.Name}"; + await Client.Mfs.CopyAsync($"/ipfs/{fileToCopy.Id}", newPath, cancel: cancellationToken); + return new MfsFile(newPath, Client); } /// @@ -69,10 +93,11 @@ public virtual async Task CreateCopyOfAsync(IpnsFile fileToCopy, boo { cancellationToken.ThrowIfCancellationRequested(); + var newPath = $"{Path}{fileToCopy.Name}"; var cid = await fileToCopy.GetCidAsync(cancellationToken); - await Client.Mfs.CopyAsync($"/ipfs/{cid}", Path, cancel: cancellationToken); + await Client.Mfs.CopyAsync($"/ipfs/{cid}", newPath, cancel: cancellationToken); - return new MfsFile($"{Path}{fileToCopy.Name}", Client); + return new MfsFile(newPath, Client); } /// @@ -80,8 +105,9 @@ public virtual async Task MoveFromAsync(MfsFile fileToMove, IModifia { cancellationToken.ThrowIfCancellationRequested(); - await Client.Mfs.MoveAsync(fileToMove.Path, $"{Path}{fileToMove.Name}", cancellationToken); - return new MfsFile($"{Path}{fileToMove.Name}", Client); + var newPath = $"{Path}{fileToMove.Name}"; + await Client.Mfs.MoveAsync(fileToMove.Path, newPath, cancellationToken); + return new MfsFile(newPath, Client); } /// diff --git a/src/MfsStream.cs b/src/MfsStream.cs index a6fab89..e6a94ae 100644 --- a/src/MfsStream.cs +++ b/src/MfsStream.cs @@ -57,7 +57,7 @@ public MfsStream(string path, long length, ICoreApi client) /// public override void Flush() { - _ = Client.Mfs.FlushAsync(Path).Result; + Client.Mfs.FlushAsync(Path).Wait(); } /// @@ -80,8 +80,7 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, var result = await Client.Mfs.ReadFileStreamAsync(Path, offset: Position, count: count, cancellationToken); var bytes = await result.ToBytesAsync(cancellationToken); - for (var i = 0; i < bytes.Length; i++) - buffer[i] = bytes[i]; + bytes.CopyTo(buffer, offset); Position += bytes.Length; @@ -108,7 +107,7 @@ public override long Seek(long offset, SeekOrigin origin) if (origin == SeekOrigin.Current) { Guard.IsLessThanOrEqualTo(Position + offset, Length); - Position = Position + offset; + Position += offset; } return Position; @@ -124,7 +123,7 @@ public override void SetLength(long value) /// public override void Write(byte[] buffer, int offset, int count) { - WriteAsync(buffer, offset, count, CancellationToken.None).GetResultOrDefault(); + WriteAsync(buffer, offset, count, CancellationToken.None).Wait(); } /// @@ -153,7 +152,8 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc SetLength(Position + count); } - await Client.Mfs.WriteAsync(Path, buffer, new() { Offset = Position, Count = count, Create = true }, cancellationToken); + await Client.Mfs.WriteAsync(Path, buffer.Skip(offset).ToArray(), new() { Offset = Position, Count = count, Create = true, Flush = false }, cancellationToken); + Position += count; } diff --git a/src/OwlCore.Kubo.csproj b/src/OwlCore.Kubo.csproj index 7402fa7..cfeb379 100644 --- a/src/OwlCore.Kubo.csproj +++ b/src/OwlCore.Kubo.csproj @@ -14,13 +14,30 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb Arlo Godfrey - 0.18.0 + 0.19.0 OwlCore An essential toolkit for Kubo, IPFS and the distributed web. LICENSE.txt +--- 0.19.0 --- +[New] +Added IAddFileToGetCid interface. This is functionally identical to IGetCid, but instead of simply returning a CID already in ipfs, it computes the CID by providing data to ipfs using preferences in the AddFileOptions parameter. + +[Breaking] +StorableKuboExtensions.GetCidAsync now takes an AddFileOptions parameter. +ContentAddressedSystemFile and ContentAddressedSystemFolder now implement IAddFileToGetCid instead of IGetCid. + +[Fixes] +Inherited fixes from OwlCore.ComponentModel 0.9.1. +Fixed issues with MfsStream where it would return before the task was complete. +MfsStream.ReadAsync and MfsStream.WriteAsync now respect the requested offset when operating on the provided buffer. + +[Improvement] +Updated to IpfsShipyard.Ipfs.Http.Client 0.5.1. +MfsStream.WriteAsync now supplies Flush = false when writing to mfs, instead of the default of flushing after every write. This improves performance when writing large files, but requires a manual call to FlushAsync to persist the changes. + --- 0.18.0 --- [Breaking] Inherited breaking changes from OwlCore.Storage 0.12.0 and OwlCore.ComponentModel 0.9.0. @@ -443,15 +460,15 @@ Added unit tests. - - - + + + - + - - + + diff --git a/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj b/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj index a312a73..247d00a 100644 --- a/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj +++ b/tests/OwlCore.Kubo.Tests/OwlCore.Kubo.Tests.csproj @@ -17,7 +17,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + runtime; build; native; contentfiles; analyzers; buildtransitive all