From f344fd0c20da9f77d6c2073082ea66b69d909340 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 14 Nov 2023 20:42:22 +0000 Subject: [PATCH] bulk copy (docs) (#70) * bulk copy * nits --- docs/bulkcopy.md | 55 ++++++++++++++++++++ docs/readme.md | 1 + test/UsageBenchmark/BatchInsertBenchmarks.cs | 10 ++-- 3 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 docs/bulkcopy.md diff --git a/docs/bulkcopy.md b/docs/bulkcopy.md new file mode 100644 index 00000000..bd862586 --- /dev/null +++ b/docs/bulkcopy.md @@ -0,0 +1,55 @@ +# Bulk Copy + +A common scenario with databases is "bulk copy"; in the case of SQL Server, this is exposed via +[`SqlBulkCopy.WriteToServer`](https://learn.microsoft.com/dotnet/api/system.data.sqlclient.sqlbulkcopy.writetoserver), +which allows a data feed to be pumped into a table very efficiently and rapidly. + +The problem is: that data feed needs to be a `DataTable`, a `DbDataReader`, or similar; but your +application code is likely to be using *objects*, for example a `List` or a `IEnumerable` sequence. + +What we need is a mechanism to turn `IEnumerable` (or `IAsyncEnumerable`) into a `DbDataReader`, +suitable for use with `SqlBulkCopy`! + +This *is not* a feature that vanilla `Dapper` provides; a library from my distant past, +[`FastMember`](https://www.nuget.org/packages/FastMember), can help bridge this gap, but like `Dapper` it is +based on runtime reflection, and is not AOT compatible. + +## Introducing `TypeAccessor` + +A new API in `Dapper.AOT` gives us what we need - welcome `TypeAccessor`! Usage is simple: + +``` csharp +IEnumerable customers = ... +var reader = TypeAccessor.CreateDataReader(customers); +``` + +This gives us a `DbDataReader` that iterates over the provided sequence. We can optionally +specify a subset of instance properties/fields to expose (and their order) - and we can start +from either `IEnumerable` or `IAsyncEnumerable`. + +To use this with `SqlBulkCopy`, we probably want to specify columns explicitly: + +``` csharp +using var table = new SqlBulkCopy(connection) +{ + DestinationTableName = "Customers", + ColumnMappings = + { + { nameof(Customer.CustomerNumber), nameof(Customer.CustomerNumber) } + { nameof(Customer.Name), nameof(Customer.Name) } + } +}; +table.EnableStreaming = true; +table.WriteToServer(TypeAccessor.CreateDataReader(customers, + [nameof(Customer.Name), nameof(Customer.CustomerNumber)])); +return table.RowsCopied; +``` + +This writes the data from our `customers` sequence into the `Customers` table. To be explicit, +we've specified the column mappings manually, although you can often omit this. Likewise, we have +restricted the `DbDataReader` to only expose the `Name` and `CustomerNumber` members. + +This all works using generated AOT-compatible code, and does not require any additional +libaries other than `Dapper.AOT`. + + diff --git a/docs/readme.md b/docs/readme.md index a8500576..6ee2f2fb 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -8,6 +8,7 @@ Index: - [Getting Started](/gettingstarted) - [SQL Syntax](/sqlsyntax) - [Generated Code](/generatedcode) +- [Bulk Copy](/bulkcopy) - [Frequently Asked Questions](/faq) Packages: diff --git a/test/UsageBenchmark/BatchInsertBenchmarks.cs b/test/UsageBenchmark/BatchInsertBenchmarks.cs index 885912c5..43106b69 100644 --- a/test/UsageBenchmark/BatchInsertBenchmarks.cs +++ b/test/UsageBenchmark/BatchInsertBenchmarks.cs @@ -14,7 +14,7 @@ namespace Dapper; public class BatchInsertBenchmarks : IDisposable { private readonly SqlConnection connection = new(Program.ConnectionString); - private Customer[] customers = Array.Empty(); + private Customer[] customers = []; public void DebugState() { @@ -246,12 +246,12 @@ public int SqlBulkCopyDapper() { DestinationTableName = "BenchmarkCustomers", ColumnMappings = - { - { nameof(Customer.Name), nameof(Customer.Name) } - } + { + { nameof(Customer.Name), nameof(Customer.Name) } + } }; table.EnableStreaming = true; - table.WriteToServer(TypeAccessor.CreateDataReader(customers, new[] { nameof(Customer.Name) })); + table.WriteToServer(TypeAccessor.CreateDataReader(customers, [nameof(Customer.Name)])); return table.RowsCopied; } finally