diff --git a/IpfsCli/Commands/ObjectCommand.cs b/IpfsCli/Commands/ObjectCommand.cs index a90aacde..d3c82a1b 100644 --- a/IpfsCli/Commands/ObjectCommand.cs +++ b/IpfsCli/Commands/ObjectCommand.cs @@ -1,12 +1,95 @@ using McMaster.Extensions.CommandLineUtils; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Text; +using System.Threading.Tasks; +using Ipfs.Engine.UnixFileSystem; +using System.IO; namespace Ipfs.Cli { - [Command(Description = "Manage raw dag nodes [WIP]")] - class ObjectCommand : CommandBase // TODO + [Command(Description = "Manage IPFS objects")] + [Subcommand("links", typeof(ObjectLinksCommand))] + [Subcommand("get", typeof(ObjectGetCommand))] + [Subcommand("dump", typeof(ObjectDumpCommand))] + class ObjectCommand : CommandBase { + public Program Parent { get; set; } + + protected override Task OnExecute(CommandLineApplication app) + { + app.ShowHelp(); + return Task.FromResult(0); + } + } + + [Command(Description = "Information on the links pointed to by the IPFS block")] + class ObjectLinksCommand : CommandBase + { + [Argument(0, "cid", "The content ID of the object")] + [Required] + public string Cid { get; set; } + + ObjectCommand Parent { get; set; } + + protected override async Task OnExecute(CommandLineApplication app) + { + var Program = Parent.Parent; + var links = await Program.CoreApi.Object.LinksAsync(Cid); + + return Program.Output(app, links, (data, writer) => + { + foreach (var link in data) + { + writer.WriteLine($"{link.Id.Encode()} {link.Size} {link.Name}"); + } + }); + } + } + + [Command(Description = "Serialise the DAG node")] + class ObjectGetCommand : CommandBase + { + [Argument(0, "cid", "The content ID of the object")] + [Required] + public string Cid { get; set; } + + ObjectCommand Parent { get; set; } + + protected override async Task OnExecute(CommandLineApplication app) + { + var Program = Parent.Parent; + var node = await Program.CoreApi.Object.GetAsync(Cid); + + return Program.Output(app, node, null); + } + } + + [Command(Description = "Dump the DAG node")] + class ObjectDumpCommand : CommandBase + { + [Argument(0, "cid", "The content ID of the object")] + [Required] + public string Cid { get; set; } + + ObjectCommand Parent { get; set; } + + class Node + { + public DagNode Dag; + public DataMessage DataMessage; + } + + protected override async Task OnExecute(CommandLineApplication app) + { + var Program = Parent.Parent; + var node = new Node(); + var block = await Program.CoreApi.Block.GetAsync(Cid); + node.Dag = new DagNode(block.DataStream); + node.DataMessage = ProtoBuf.Serializer.Deserialize(node.Dag.DataStream); + + return Program.Output(app, node, null); + } } } diff --git a/IpfsCli/Program.cs b/IpfsCli/Program.cs index c3b7a652..8050efc7 100644 --- a/IpfsCli/Program.cs +++ b/IpfsCli/Program.cs @@ -90,7 +90,7 @@ public static int Main(string[] args) } var took = DateTime.Now - startTime; - Console.Write($"Took {took.TotalSeconds} seconds."); + //Console.Write($"Took {took.TotalSeconds} seconds."); return 0; } diff --git a/src/CoreApi/FileSystemApi.cs b/src/CoreApi/FileSystemApi.cs index 1d211534..f820acd1 100644 --- a/src/CoreApi/FileSystemApi.cs +++ b/src/CoreApi/FileSystemApi.cs @@ -18,6 +18,8 @@ class FileSystemApi : IFileSystemApi static ILog log = LogManager.GetLogger(typeof(FileSystemApi)); IpfsEngine ipfs; + static readonly int DefaultLinksPerBlock = 174; + public FileSystemApi(IpfsEngine ipfs) { this.ipfs = ipfs; @@ -63,42 +65,7 @@ public async Task AddAsync( var nodes = await chunker.ChunkAsync(stream, name, options, blockService, keyChain, cancel).ConfigureAwait(false); // Multiple nodes for the file? - FileSystemNode node = null; - if (nodes.Count() == 1) - { - node = nodes.First(); - } - else - { - // Build the DAG that contains all the file nodes. - var links = nodes.Select(n => n.ToLink()).ToArray(); - var fileSize = (ulong)nodes.Sum(n => n.Size); - var dm = new DataMessage - { - Type = DataType.File, - FileSize = fileSize, - BlockSizes = nodes.Select(n => (ulong) n.Size).ToArray() - }; - var pb = new MemoryStream(); - ProtoBuf.Serializer.Serialize(pb, dm); - var dag = new DagNode(pb.ToArray(), links, options.Hash); - - // Save it. - dag.Id = await blockService.PutAsync( - data: dag.ToArray(), - multiHash: options.Hash, - encoding: options.Encoding, - pin: options.Pin, - cancel: cancel).ConfigureAwait(false); - - node = new FileSystemNode - { - Id = dag.Id, - Size = (long)dm.FileSize, - DagSize = dag.Size, - Links = links - }; - } + FileSystemNode node = await BuildTreeAsync(nodes, options, cancel); // Wrap in directory? if (options.Wrap) @@ -122,6 +89,71 @@ public async Task AddAsync( return node; } + async Task BuildTreeAsync( + IEnumerable nodes, + AddFileOptions options, + CancellationToken cancel) + { + if (nodes.Count() == 1) + { + return nodes.First(); + } + + // Bundle DefaultLinksPerBlock links into a block. + var tree = new List(); + for (int i = 0; true; ++i) + { + var bundle = nodes + .Skip(DefaultLinksPerBlock * i) + .Take(DefaultLinksPerBlock); + if (bundle.Count() == 0) + { + break; + } + var node = await BuildTreeNodeAsync(bundle, options, cancel); + tree.Add(node); + } + return await BuildTreeAsync(tree, options, cancel); + } + + async Task BuildTreeNodeAsync( + IEnumerable nodes, + AddFileOptions options, + CancellationToken cancel) + { + var blockService = GetBlockService(options); + + // Build the DAG that contains all the file nodes. + var links = nodes.Select(n => n.ToLink()).ToArray(); + var fileSize = (ulong)nodes.Sum(n => n.Size); + var dagSize = nodes.Sum(n => n.DagSize); + var dm = new DataMessage + { + Type = DataType.File, + FileSize = fileSize, + BlockSizes = nodes.Select(n => (ulong)n.Size).ToArray() + }; + var pb = new MemoryStream(); + ProtoBuf.Serializer.Serialize(pb, dm); + var dag = new DagNode(pb.ToArray(), links, options.Hash); + + // Save it. + dag.Id = await blockService.PutAsync( + data: dag.ToArray(), + multiHash: options.Hash, + encoding: options.Encoding, + pin: options.Pin, + cancel: cancel).ConfigureAwait(false); + + return new FileSystemNode + { + Id = dag.Id, + Size = (long)dm.FileSize, + DagSize = dagSize + dag.Size, + Links = links + }; + } + public async Task AddDirectoryAsync( string path, bool recursive = true, diff --git a/src/UnixFileSystem/DataMessage.cs b/src/UnixFileSystem/DataMessage.cs index 99c6d642..539c4dde 100644 --- a/src/UnixFileSystem/DataMessage.cs +++ b/src/UnixFileSystem/DataMessage.cs @@ -10,13 +10,39 @@ namespace Ipfs.Engine.UnixFileSystem { - enum DataType + /// + /// Specifies the type of data. + /// + public enum DataType { + /// + /// Raw data + /// Raw = 0, + + /// + /// A directory of files. + /// Directory = 1, + + /// + /// A file. + /// File = 2, + + /// + /// Metadata (NYI) + /// Metadata = 3, + + /// + /// Symbolic link (NYI) + /// Symlink = 4, + + /// + /// NYI + /// HAMTShard = 5 }; @@ -24,25 +50,43 @@ enum DataType /// The ProtoBuf data that is stored in a DAG. /// [ProtoContract] - internal class DataMessage + public class DataMessage { + /// + /// The type of data. + /// [ProtoMember(1, IsRequired = true)] public DataType Type; + /// + /// The data. + /// [ProtoMember(2, IsRequired = false)] public byte[] Data; + /// + /// The file size. + /// [ProtoMember(3, IsRequired = false)] public ulong? FileSize; + /// + /// The file size of each block. + /// [ProtoMember(4, IsRequired = false)] public ulong[] BlockSizes; #pragma warning disable 0649 // disable warning about unassinged fields + /// + /// NYI + /// [ProtoMember(5, IsRequired = false)] public ulong? HashType; #pragma warning disable 0649 // disable warning about unassinged fields + /// + /// NYI + /// [ProtoMember(6, IsRequired = false)] public ulong? Fanout; } diff --git a/src/UnixFileSystem/FileSystem.cs b/src/UnixFileSystem/FileSystem.cs index 05beeed7..503b89ac 100644 --- a/src/UnixFileSystem/FileSystem.cs +++ b/src/UnixFileSystem/FileSystem.cs @@ -16,7 +16,7 @@ namespace Ipfs.Engine.UnixFileSystem /// public static class FileSystem { - static byte[] emptyData = new byte[0]; + static readonly byte[] emptyData = new byte[0]; /// /// Creates a stream that can read the supplied . diff --git a/src/UnixFileSystem/SizeChunker.cs b/src/UnixFileSystem/SizeChunker.cs index 1c304846..6c481cb0 100644 --- a/src/UnixFileSystem/SizeChunker.cs +++ b/src/UnixFileSystem/SizeChunker.cs @@ -40,7 +40,7 @@ public class SizeChunker /// A task that represents the asynchronous operation. The task's value is /// the sequence of file system nodes of the added data blocks. /// - public async Task> ChunkAsync( + public async Task> ChunkAsync( Stream stream, string name, AddFileOptions options, diff --git a/test/CoreApi/FileSystemApiTest.cs b/test/CoreApi/FileSystemApiTest.cs index cfb124b1..263afe22 100644 --- a/test/CoreApi/FileSystemApiTest.cs +++ b/test/CoreApi/FileSystemApiTest.cs @@ -228,6 +228,48 @@ public void AddFile_Large() Console.WriteLine("Readfile file took {0} seconds.", stopWatch.Elapsed.TotalSeconds); } + + /// + [TestMethod] + public void AddFile_Larger() + { + AddFile(); // warm up + + var path = "starx2.mp4"; + var ipfs = TestFixture.Ipfs; + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var node = ipfs.FileSystem.AddFileAsync(path).Result; + stopWatch.Stop(); + Console.WriteLine("Add file took {0} seconds.", stopWatch.Elapsed.TotalSeconds); + + Assert.AreEqual("QmeFhfB4g2GFbxYb7usApWzq8uC1vmuxJajFpiJiT5zLoy", (string)node.Id); + + var k = 8 * 1024; + var buffer1 = new byte[k]; + var buffer2 = new byte[k]; + stopWatch.Restart(); + using (var localStream = new FileStream(path, FileMode.Open, FileAccess.Read)) + using (var ipfsStream = ipfs.FileSystem.ReadFileAsync(node.Id).Result) + { + while (true) + { + var n1 = localStream.Read(buffer1, 0, k); + var n2 = ipfsStream.Read(buffer2, 0, k); + Assert.AreEqual(n1, n2); + if (n1 == 0) + break; + for (var i = 0; i < n1; ++i) + { + if (buffer1[i] != buffer2[i]) + Assert.Fail("data not the same"); + } + } + } + stopWatch.Stop(); + Console.WriteLine("Readfile file took {0} seconds.", stopWatch.Elapsed.TotalSeconds); + } + [TestMethod] public async Task AddFile_Wrap() { diff --git a/test/IpfsEngineTests.csproj b/test/IpfsEngineTests.csproj index 6f2f844d..f7c684a8 100644 --- a/test/IpfsEngineTests.csproj +++ b/test/IpfsEngineTests.csproj @@ -33,6 +33,9 @@ + + Always + Always diff --git a/test/starx2.mp4 b/test/starx2.mp4 new file mode 100644 index 00000000..101c3506 Binary files /dev/null and b/test/starx2.mp4 differ