diff --git a/NetStandard.SqlBulkHelpers/Database/TableNameTerm.cs b/NetStandard.SqlBulkHelpers/Database/TableNameTerm.cs index 4700e92..5f00402 100644 --- a/NetStandard.SqlBulkHelpers/Database/TableNameTerm.cs +++ b/NetStandard.SqlBulkHelpers/Database/TableNameTerm.cs @@ -23,10 +23,19 @@ public TableNameTerm(string schemaName, string tableName) public string FullyQualifiedTableName { get; } public override string ToString() => FullyQualifiedTableName; - public bool Equals(TableNameTerm other) => FullyQualifiedTableName.Equals(other.FullyQualifiedTableName); - public bool EqualsIgnoreCase(TableNameTerm other) => FullyQualifiedTableName.Equals(other.FullyQualifiedTableName, StringComparison.OrdinalIgnoreCase); - public TableNameTerm SwitchSchema(string newSchema) => new TableNameTerm(newSchema, TableName); - + + public bool Equals(TableNameTerm other) + => FullyQualifiedTableName.Equals(other.FullyQualifiedTableName); + + public bool EqualsIgnoreCase(TableNameTerm other) + => FullyQualifiedTableName.Equals(other.FullyQualifiedTableName, StringComparison.OrdinalIgnoreCase); + + public TableNameTerm SwitchSchema(string newSchema) + => new TableNameTerm(newSchema, TableName); + + public TableNameTerm ApplyNamePrefixOrSuffix(string prefix = null, string suffix = null) + => new TableNameTerm(SchemaName, string.Concat(prefix?.Trim() ?? string.Empty, TableName, suffix?.Trim() ?? string.Empty)); + //Handle Automatic String conversions for simplified APIs... public static implicit operator string(TableNameTerm t) => t.ToString(); diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/CloneTableInfo.cs b/NetStandard.SqlBulkHelpers/MaterializedData/CloneTableInfo.cs index 21d9ada..563bbba 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/CloneTableInfo.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/CloneTableInfo.cs @@ -30,9 +30,9 @@ public CloneTableInfo MakeTargetTableNameUnique() => new CloneTableInfo(SourceTable, MakeTableNameUniqueInternal(TargetTable)); private static TableNameTerm MakeTableNameUniqueInternal(TableNameTerm tableNameTerm) - => TableNameTerm.From(tableNameTerm.SchemaName, $"{tableNameTerm.TableName}_Copy_{IdGenerator.NewId(10)}"); + => TableNameTerm.From(tableNameTerm.SchemaName, string.Concat(tableNameTerm.TableName, "_", IdGenerator.NewId(10))); - public static CloneTableInfo From(string sourceTableName = null, string targetTableName = null) + public static CloneTableInfo From(string sourceTableName = null, string targetTableName = null, string targetPrefix = null, string targetSuffix = null) { //If the generic type is ISkipMappingLookup then we must have a valid sourceTableName specified as a param... if (SqlBulkHelpersProcessingDefinition.SkipMappingLookupType.IsAssignableFrom(typeof(TSource))) @@ -46,7 +46,7 @@ public static CloneTableInfo From(string sourceTableName = nul ? sourceTableName : targetTableName; - //If the generic type is ISkipMappingLookup then we must have a valid sourceTableName specified as a param... + //If the generic type is ISkipMappingLookup then we must have a valid validTargetTableName specified as a param... if (SqlBulkHelpersProcessingDefinition.SkipMappingLookupType.IsAssignableFrom(typeof(TTarget))) { //We validate the valid target table name but if it's still blank then we throw an Argument @@ -54,14 +54,14 @@ public static CloneTableInfo From(string sourceTableName = nul validTargetTableName.AssertArgumentIsNotNullOrWhiteSpace(nameof(targetTableName)); } - var targetTable = TableNameTerm.From(targetTableName ?? sourceTableName); + var targetTable = TableNameTerm.From(targetTableName ?? sourceTableName).ApplyNamePrefixOrSuffix(targetPrefix, targetSuffix); return new CloneTableInfo(sourceTable, targetTable); } - public static CloneTableInfo From(string sourceTableName, string targetTableName = null) - => From(sourceTableName, targetTableName); + public static CloneTableInfo From(string sourceTableName, string targetTableName = null, string targetPrefix = null, string targetSuffix = null) + => From(sourceTableName, targetTableName, targetPrefix, targetSuffix); - public static CloneTableInfo ForNewSchema(TableNameTerm sourceTable, string targetSchemaName) - => new CloneTableInfo(sourceTable, sourceTable.SwitchSchema(targetSchemaName)); + public static CloneTableInfo ForNewSchema(TableNameTerm sourceTable, string targetSchemaName, string targetTablePrefix = null, string targetTableSuffix = null) + => new CloneTableInfo(sourceTable, sourceTable.SwitchSchema(targetSchemaName).ApplyNamePrefixOrSuffix(targetTablePrefix, targetTableSuffix)); } } diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContextCompletionSource.cs b/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContextCompletionSource.cs index 2192b7e..b15ed8a 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContextCompletionSource.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContextCompletionSource.cs @@ -1,9 +1,10 @@ using System.Threading.Tasks; +using Microsoft.Data.SqlClient; namespace SqlBulkHelpers.MaterializedData { public interface IMaterializeDataContextCompletionSource : IMaterializeDataContext { - Task FinishMaterializeDataProcessAsync(); + Task FinishMaterializeDataProcessAsync(SqlTransaction sqlTransaction); } } \ No newline at end of file diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs index dd2c438..c80d604 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs @@ -9,7 +9,6 @@ namespace SqlBulkHelpers.MaterializedData { public class MaterializeDataContext : IMaterializeDataContextCompletionSource, IMaterializeDataContext { - protected SqlTransaction SqlTransaction { get; } protected ILookup TableLookupByFullyQualifiedName { get; } protected ILookup TableLookupByOriginalName { get; } protected ISqlBulkHelpersConfig BulkHelpersConfig { get; } @@ -80,16 +79,15 @@ public MaterializationTableInfo FindMaterializationTableInfoCaseInsensitive(Type /// public bool EnableDataConstraintChecksOnCompletion { get; set; } = true; - public MaterializeDataContext(SqlTransaction sqlTransaction, MaterializationTableInfo[] materializationTables, ISqlBulkHelpersConfig bulkHelpersConfig) + public MaterializeDataContext(MaterializationTableInfo[] materializationTables, ISqlBulkHelpersConfig bulkHelpersConfig) { - SqlTransaction = sqlTransaction.AssertArgumentIsNotNull(nameof(sqlTransaction)); Tables = materializationTables.AssertArgumentIsNotNull(nameof(materializationTables)); BulkHelpersConfig = bulkHelpersConfig.AssertArgumentIsNotNull(nameof(bulkHelpersConfig)); TableLookupByFullyQualifiedName = Tables.ToLookup(t => t.LiveTable.FullyQualifiedTableName, StringComparer.OrdinalIgnoreCase); TableLookupByOriginalName = Tables.ToLookup(t => t.OriginalTableName, StringComparer.OrdinalIgnoreCase); } - internal async Task HandleNonTransactionTasksBeforeMaterialization() + internal async Task HandleParallelConnectionTasksBeforeMaterialization() { //NOW JUST PRIOR to Executing the Materialized Data Switch we must handle any actions required outside of the Materialized Data Transaction (e.g. FullTextIndexes, etc.) //NOTE: We do this here so that our live tables have the absolute minimum impact; meaning things like Full Text Indexes are Dropped for ONLY the amount of time it takes to execute our Switch @@ -118,7 +116,7 @@ await tablesWithFullTextIndexes.ForEachAsync(BulkHelpersConfig.MaxConcurrentConn /// occurs during the Finish Materialization process!!! /// /// - internal async Task HandleNonTransactionTasksAfterMaterialization() + internal async Task HandleParallelConnectionTasksAfterMaterialization() { if (TablesWithFullTextIndexesRemoved.Any()) { @@ -134,7 +132,7 @@ await TablesWithFullTextIndexesRemoved.ForEachAsync(BulkHelpersConfig.MaxConcurr } } - public async Task FinishMaterializeDataProcessAsync() + public async Task FinishMaterializeDataProcessAsync(SqlTransaction sqlTransaction) { var materializationTables = this.Tables; var switchScriptBuilder = MaterializedDataScriptBuilder.NewSqlScript(); @@ -207,16 +205,10 @@ public async Task FinishMaterializeDataProcessAsync() //NOTE: FKeys must be explicitly re-enabled to ensure they are restored to Trusted state; they aren't included in the ALL Constraint Check. .EnableForeignKeyChecks(materializationTableInfo.LiveTable, this.EnableDataConstraintChecksOnCompletion, liveTableDefinition.ForeignKeyConstraints.AsArray()) //Re-enable All other Referencing FKey Checks that were disable above to allow the switching above... - .EnableReferencingForeignKeyChecks(this.EnableDataConstraintChecksOnCompletion, otherReferencingFKeyConstraints.AsArray()) - //Finally cleanup the Loading and Discarding tables... - .DropTable(materializationTableInfo.LoadingTable) - .DropTable(materializationTableInfo.DiscardingTable); - + .EnableReferencingForeignKeyChecks(this.EnableDataConstraintChecksOnCompletion, otherReferencingFKeyConstraints.AsArray()); } - //var timeoutConvertedToMinutes = Math.Max(1, (int)Math.Ceiling((decimal)BulkHelpersConfig.MaterializeDataStructureProcessingTimeoutSeconds / 60)); - - await SqlTransaction.ExecuteMaterializedDataSqlScriptAsync( + await sqlTransaction.ExecuteMaterializedDataSqlScriptAsync( switchScriptBuilder, BulkHelpersConfig.MaterializeDataStructureProcessingTimeoutSeconds ).ConfigureAwait(false); diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs index 8ebdb5f..846852a 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs @@ -1,11 +1,10 @@ -using System; +using Microsoft.Data.SqlClient; +using SqlBulkHelpers.CustomExtensions; +using System; using System.Collections.Generic; using System.Data; -using System.Data.SqlTypes; using System.Linq; using System.Threading.Tasks; -using Microsoft.Data.SqlClient; -using SqlBulkHelpers.CustomExtensions; namespace SqlBulkHelpers.MaterializedData { @@ -23,22 +22,78 @@ public MaterializeDataHelper(ISqlBulkHelpersConfig bulkHelpersConfig = null) #region Materialize Data API Methods (& Cloning for Materialization Process) + /// + /// This Method will copy the Schema within the specified transaction which will result in a Schema lock for the duration of the Transaction; + /// therefore it's recommended to use a separate SqlTransaction from the actual loading process just for initialization of the Loading Schema (copied tables). + /// And then use a separate Sql Transaction in a Try/Finally to Clean up the Materialization process via CleanupMaterializeDataProcessAsync(). + /// + /// + /// + /// public Task StartMaterializeDataProcessAsync(SqlTransaction sqlTransaction, params string[] tableNames) => StartMaterializeDataProcessAsync(sqlTransaction, BulkHelpersConfig.MaterializedDataLoadingSchema, BulkHelpersConfig.MaterializedDataDiscardingSchema, tableNames); + /// + /// This Method will copy the Schema within the specified transaction which will result in a Schema lock for the duration of the Transaction; + /// therefore it's recommended to use a separate SqlTransaction from the actual loading process just for initialization of the Loading Schema (copied tables). + /// And then use a separate Sql Transaction in a Try/Finally to Clean up the Materialization process via CleanupMaterializeDataProcessAsync(). + /// + /// + /// + /// + /// + /// public async Task StartMaterializeDataProcessAsync(SqlTransaction sqlTransaction, string loadingSchemaName, string discardingSchemaName, params string[] tableNames) { //This will clone each of the Live tables into one Temp and one Loading table (each in different Schema as defined in the BulkHelperSettings).. - var cloneMaterializationTables = await CloneTableStructuresForMaterializationAsync( + var cloneMaterializationTables = await CloneTableStructuresForMaterializationInternalAsync( sqlTransaction, tableNames, loadingSchemaName, discardingSchemaName ).ConfigureAwait(false); - return new MaterializeDataContext(sqlTransaction, cloneMaterializationTables, this.BulkHelpersConfig); + return new MaterializeDataContext(cloneMaterializationTables, this.BulkHelpersConfig); } + /// + /// This Method will clean up the the Loading/Discarding Schema, as defined in the MaterializedDataContext, specified transaction which will result in a Schema lock for the duration of the Transaction; + /// therefore it's recommended to use a separate SqlTransaction from the actual loading process just for initialization of the Loading Schema (copied tables). + /// And then use a separate Sql Transaction in a Try/Finally to Clean up the Materialization process via CleanupMaterializeDataProcessAsync(). + /// + /// + /// + /// + public async Task CleanupMaterializeDataProcessAsync(SqlTransaction sqlTransaction, IMaterializeDataContext materializedDataContext) + { + var materializationTables = materializedDataContext.Tables; + var switchScriptBuilder = MaterializedDataScriptBuilder.NewSqlScript(); + + //Explicitly clean up all Loading/Discarding Tables (contains old data) to free resources -- this leaves us with only the (new) Live Table in place! + foreach (var materializationTableInfo in materializationTables) + { + switchScriptBuilder + //Finally cleanup the Loading and Discarding tables... + .DropTableIfExists(materializationTableInfo.LoadingTable) + .DropTableIfExists(materializationTableInfo.DiscardingTable); + } + + await sqlTransaction.ExecuteMaterializedDataSqlScriptAsync( + switchScriptBuilder, + BulkHelpersConfig.MaterializeDataStructureProcessingTimeoutSeconds + ).ConfigureAwait(false); + + return materializedDataContext; + } + + /// + /// This Method will copy the Schema within the specified transaction which will result in a Schema lock for the duration of the Transaction; + /// therefore it's recommended to use a separate SqlTransaction from the actual loading process just for initialization of the Loading Schema (copied tables). + /// And then use a separate Sql Transaction in a Try/Finally to Clean up the Materialization process via CleanupMaterializeDataProcessAsync(). + /// + /// + /// + /// public Task CloneTableStructuresForMaterializationAsync( SqlTransaction sqlTransaction, params string[] tableNames @@ -49,10 +104,32 @@ params string[] tableNames BulkHelpersConfig.MaterializedDataDiscardingSchema ); - public async Task CloneTableStructuresForMaterializationAsync( - SqlTransaction sqlTransaction, - IEnumerable tableNames, - string loadingSchemaName = null, + /// + /// This Method will copy the Schema within the specified transaction which will result in a Schema lock for the duration of the load; + /// it's recommended to use the overload with an SqlConnection instead which will initialize it's own SqlTransaction (as part of the MaterializedDataContext) + /// after the Schema copy is complete so Schema locks are avoided but Try/Finally MUST be used to ensure the Schema is cleaned up! + /// + /// + /// + /// + /// + /// + public Task CloneTableStructuresForMaterializationAsync( + SqlTransaction sqlTransaction, + IEnumerable tableNames, + string loadingSchemaName = null, + string discardingSchemaName = null + ) => CloneTableStructuresForMaterializationInternalAsync( + sqlTransaction, + tableNames, + loadingSchemaName, + discardingSchemaName + ); + + protected async Task CloneTableStructuresForMaterializationInternalAsync( + SqlTransaction sqlTransaction, + IEnumerable tableNames, + string loadingSchemaName = null, string discardingSchemaName = null ) { @@ -74,7 +151,7 @@ public async Task CloneTableStructuresForMaterializa //Optional Perf. Optimization: If ConcurrentConnections are enabled we can optimize performance by asynchronously pre-loading Table Schemas with concurrent Sql Connections... if (BulkHelpersConfig.IsConcurrentConnectionProcessingEnabled) - await PreCacheTableSchemaDefinitionsForMaterialization(tableNameTermsList).ConfigureAwait(false); + await PreCacheTableSchemaDefinitionsForMaterializationAsync(tableNameTermsList).ConfigureAwait(false); //1) First compute all table cloning instructions, and Materialization table info./details to generate the Loading Tables and the Discard Tables for every table to be cloned... foreach (var originalTableNameTerm in tableNameTermsList) @@ -82,20 +159,31 @@ public async Task CloneTableStructuresForMaterializa //Add Clones for Loading tables... //NOTE: We make the Loading table name highly unique just in case multiple processes run at the same time they will have less risk of impacting each other; // though such a conflict would be a flawed design and should be eliminated via an SQL lock or Distributed Mutex lock (aka SqlAppLockHelper library). - var loadingCloneInfo = CloneTableInfo.ForNewSchema(originalTableNameTerm, loadingTablesSchema).MakeTargetTableNameUnique(); - cloneInfoToExecuteList.Add(loadingCloneInfo); - - // ReSharper disable once PossibleNullReferenceException - bool isDiscardingSchemaDifferentFromLoadingSchema = !loadingSchemaName.Equals(discardingTablesSchema, StringComparison.OrdinalIgnoreCase); + var loadingCloneInfo = CloneTableInfo.ForNewSchema( + originalTableNameTerm, + loadingTablesSchema, + BulkHelpersConfig.MaterializedDataLoadingTablePrefix, + BulkHelpersConfig.MaterializedDataLoadingTableSuffix + ); //Add Clones for Discarding tables (used for switching Live OUT for later cleanup)... //NOTE: We try to keep our Loading and Discarding table names in sync if possible but enforce their uniqueness if the Schema names are not different... - var discardingCloneInfo = isDiscardingSchemaDifferentFromLoadingSchema - //Try to keep the Table Names highly unique but in-sync between Loading and Discarding schemas (for debugging purposes mainly). - ? CloneTableInfo.From(originalTableNameTerm, loadingCloneInfo.TargetTable.SwitchSchema(discardingTablesSchema)) - //Otherwise enforce uniqueness... - : CloneTableInfo.ForNewSchema(originalTableNameTerm, discardingTablesSchema).MakeTargetTableNameUnique(); - + var discardingCloneInfo = CloneTableInfo.ForNewSchema( + originalTableNameTerm, + discardingTablesSchema, + BulkHelpersConfig.MaterializedDataDiscardingTablePrefix, + BulkHelpersConfig.MaterializedDataDiscardingTableSuffix + ); + + //Enforce table name uniqueness if necessary and enabled in configuration... + if (BulkHelpersConfig.MaterializedDataMakeSchemaCopyNamesUnique) + { + loadingCloneInfo = loadingCloneInfo.MakeTargetTableNameUnique(); + discardingCloneInfo = discardingCloneInfo.MakeTargetTableNameUnique(); + } + + //Add to our List to be processed... + cloneInfoToExecuteList.Add(loadingCloneInfo); cloneInfoToExecuteList.Add(discardingCloneInfo); //Finally aggregate the Live/Original, Loading, and Discarding tables into the MaterializationTableInfo @@ -153,7 +241,7 @@ protected void AssertTableSchemaDefinitionAndConfigurationIsValidForMaterializat /// /// /// - protected async Task> PreCacheTableSchemaDefinitionsForMaterialization(IEnumerable tableNameTerms) + protected async Task> PreCacheTableSchemaDefinitionsForMaterializationAsync(IEnumerable tableNameTerms) { var tableDefinitionResults = new List(); @@ -218,6 +306,7 @@ protected async Task CloneTablesInternalAsync( ) { sqlTransaction.AssertArgumentIsNotNull(nameof(sqlTransaction)); + var cloneInfoList = tablesToClone.ToList(); if (cloneInfoList.IsNullOrEmpty()) @@ -281,7 +370,7 @@ public async Task DropTablesAsync(SqlTransaction sqlTransaction var sqlScriptBuilder = MaterializedDataScriptBuilder.NewSqlScript(); foreach (var tableNameTerm in tableNameTermsList) - sqlScriptBuilder.DropTable(tableNameTerm); + sqlScriptBuilder.DropTableIfExists(tableNameTerm); //Execute the Script! await sqlTransaction @@ -317,9 +406,9 @@ public async Task ClearTablesAsync(SqlTransaction sqlTransactio var tableDef = await GetTableSchemaDefinitionInternalAsync(TableSchemaDetailLevel.ExtendedDetails, sqlConnection, sqlTransaction: sqlTransaction, tableNameTerm).ConfigureAwait(false); //Bucket our Table Definitions based on if they REQUIRE Materialization or if they can be handled by Truncate processing... if (tableDef.ReferencingForeignKeyConstraints.HasAny() || tableDef.ForeignKeyConstraints.HasAny()) - lock (tablesToMaterializeAsEmpty) tablesToMaterializeAsEmpty.Add(tableNameTerm); + tablesToMaterializeAsEmpty.Add(tableNameTerm); else - lock (tablesToProcessWithTruncation) tablesToProcessWithTruncation.Add(tableNameTerm); + tablesToProcessWithTruncation.Add(tableNameTerm); } if (tablesToMaterializeAsEmpty.Any()) @@ -329,9 +418,15 @@ public async Task ClearTablesAsync(SqlTransaction sqlTransactio // and we simply complete the process by materializing to EMPTY tables (newly cloned) with no data! //START the Materialize Data Process... but we do NOT insert any new data to the Empty Tables! var materializeDataContext = await sqlTransaction.StartMaterializeDataProcessAsync(tablesToMaterializeAsEmpty).ConfigureAwait(false); - - //We finish the Clearing process by immediately switching out with the new/empty tables to Clear the Data! - await materializeDataContext.FinishMaterializeDataProcessAsync().ConfigureAwait(false); + try + { + //We finish the Clearing process by immediately switching out with the new/empty tables to Clear the Data! + await materializeDataContext.FinishMaterializeDataProcessAsync(sqlTransaction).ConfigureAwait(false); + } + finally + { + await CleanupMaterializeDataProcessAsync(sqlTransaction, materializeDataContext).ConfigureAwait(false); + } } } else @@ -345,7 +440,7 @@ public async Task ClearTablesAsync(SqlTransaction sqlTransactio var truncateTableSqlScriptBuilder = MaterializedDataScriptBuilder.NewSqlScript(); foreach (var tableNameTerm in tablesToProcessWithTruncation) - truncateTableSqlScriptBuilder.TruncateTable(tableNameTerm); + truncateTableSqlScriptBuilder.TruncateTableIfExists(tableNameTerm); //Execute the Script! await sqlTransaction @@ -492,8 +587,9 @@ private SqlCommand CreateReSeedTableIdentityValueToSyncWithMaxIdSqlCommand(SqlCo var tableNameTerm = tableDef.TableNameTerm; var maxIdVariable = $"@MaxId_{tableNameTerm.TableNameVariable}"; + //NOTE: If the Table is EMPTY then MAX(Id) will return null so we must Coalesce to the Default value of 1! var sqlCmd = new SqlCommand($@" - DECLARE {maxIdVariable} BIGINT = (SELECT MAX({tableDef.IdentityColumn.ColumnName.QualifySqlTerm()}) FROM {tableNameTerm.FullyQualifiedTableName}); + DECLARE {maxIdVariable} BIGINT = (SELECT COALESCE(MAX({tableDef.IdentityColumn.ColumnName.QualifySqlTerm()}), 1) FROM {tableNameTerm.FullyQualifiedTableName}); DBCC CHECKIDENT(@TableName, RESEED, {maxIdVariable}); SELECT CURRENT_IDENTITY_VALUE = IDENT_CURRENT(@TableName); ", sqlConnection, sqlTransaction); diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataScriptBuilder.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataScriptBuilder.cs index da1d2f4..47e6e1c 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataScriptBuilder.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataScriptBuilder.cs @@ -52,7 +52,7 @@ If SCHEMA_ID('{sanitizedSchemaName}') IS NULL return this; } - public MaterializedDataScriptBuilder DropTable(TableNameTerm tableName) + public MaterializedDataScriptBuilder DropTableIfExists(TableNameTerm tableName) { tableName.AssertArgumentIsNotNull(nameof(tableName)); ScriptBuilder.Append($@" @@ -63,7 +63,7 @@ IF OBJECT_ID('{tableName.FullyQualifiedTableName}') IS NOT NULL return this; } - public MaterializedDataScriptBuilder TruncateTable(TableNameTerm tableName) + public MaterializedDataScriptBuilder TruncateTableIfExists(TableNameTerm tableName) { tableName.AssertArgumentIsNotNull(nameof(tableName)); ScriptBuilder.Append($@" @@ -212,7 +212,7 @@ public MaterializedDataScriptBuilder CloneTableWithColumnsOnly(TableNameTerm sou if (ifExists == IfExists.Recreate) { addTableCopyScript = true; - DropTable(targetTable); + DropTableIfExists(targetTable); } else if (ifExists == IfExists.StopProcessingWithException) { diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataSqlClientExtensionsApi.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataSqlClientExtensionsApi.cs index 323fd19..ad962e1 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataSqlClientExtensionsApi.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializedDataSqlClientExtensionsApi.cs @@ -532,57 +532,114 @@ public static async Task ExecuteMaterializeDataProcessAsync( var distinctTableNames = tableNames.Distinct().ToArray(); - //SECOND Execute the Materialized Data process! - #if NETSTANDARD2_0 - using (var sqlTransaction = sqlConnection.BeginTransaction()) - #else - await using (var sqlTransaction = (SqlTransaction)await sqlConnection.BeginTransactionAsync().ConfigureAwait(false)) - #endif - { - var materializeDataContext = await new MaterializeDataHelper(sqlBulkHelpersConfig) - .StartMaterializeDataProcessAsync(sqlTransaction, distinctTableNames) - .ConfigureAwait(false); - - //The Handler is always an Async process so we must await it... - await materializeDataHandlerActionAsync.Invoke(materializeDataContext, sqlTransaction).ConfigureAwait(false); + var materializationDataHelper = new MaterializeDataHelper(bulkHelpersConfig); + MaterializeDataContext materializeDataContext = null; - //Enable Passive cancellation vs throwing an Exception for advanced use cases... - if (materializeDataContext.IsCancelled) + //Initialize our Schema outside the transaction to avoid Schema locks (now Default Behaviour as of v2.3.0)... + if (sqlBulkHelpersConfig.MaterializedDataSchemaCopyMode == SchemaCopyMode.OutsideTransactionAvoidSchemaLocks) + { + #if NETSTANDARD2_0 + using (var sqlTransaction = sqlConnection.BeginTransaction()) + #else + await using (var sqlTransaction = (SqlTransaction)await sqlConnection.BeginTransactionAsync().ConfigureAwait(false)) + #endif { - //NOW we must commit our Transaction to save all Changes performed within the Transaction! + materializeDataContext = await materializationDataHelper + .StartMaterializeDataProcessAsync(sqlTransaction, distinctTableNames) + .ConfigureAwait(false); + #if NETSTANDARD2_0 - sqlTransaction.Rollback(); + sqlTransaction.Commit(); #else - await sqlTransaction.RollbackAsync().ConfigureAwait(false); + await sqlTransaction.CommitAsync().ConfigureAwait(false); #endif } - else + } + + try + { + //SECOND Execute the Materialized Data process! + #if NETSTANDARD2_0 + using (var sqlTransaction = sqlConnection.BeginTransaction()) + #else + await using (var sqlTransaction = (SqlTransaction)await sqlConnection.BeginTransactionAsync().ConfigureAwait(false)) + #endif { - try + + //If configured (not already handled above) then we initialize our Schema inside the transaction... + if (sqlBulkHelpersConfig.MaterializedDataSchemaCopyMode == SchemaCopyMode.InsideTransactionAllowSchemaLocks) { - //Some tasks MUST be done outside of the Materialized Data Transaction (e.g. handling FullTextIndexes) - // so we handle those here (as needed and if enabled)... - await materializeDataContext.HandleNonTransactionTasksBeforeMaterialization().ConfigureAwait(false); + materializeDataContext = await materializationDataHelper + .StartMaterializeDataProcessAsync(sqlTransaction, distinctTableNames) + .ConfigureAwait(false); + } + + if (materializeDataContext is null) + throw new InvalidOperationException("The Materialized Data Context is null but must be fully initialized before processing may continue."); - //*************************************************************************************************** - //****HERE We actually Execute the Materialized Data Processing SWITCH and Data Integrity Checks! - //*************************************************************************************************** - //Once completed without errors we Finish the Materialized Data process... - await materializeDataContext.FinishMaterializeDataProcessAsync().ConfigureAwait(false); + //The Handler is always an Async process so we must await it... + await materializeDataHandlerActionAsync.Invoke(materializeDataContext, sqlTransaction).ConfigureAwait(false); + + //Enable Passive cancellation vs throwing an Exception for advanced use cases... + if (!materializeDataContext.IsCancelled) + { + try + { + //Some tasks MUST be done outside of the Materialized Data Transaction (e.g. handling FullTextIndexes) + // so we handle those here (as needed and if enabled)... + await materializeDataContext.HandleParallelConnectionTasksBeforeMaterialization().ConfigureAwait(false); + + //*************************************************************************************************** + //****HERE We actually Execute the Materialized Data Processing SWITCH and Data Integrity Checks! + //*************************************************************************************************** + //Once completed without errors we Finish the Materialized Data process... + await materializeDataContext.FinishMaterializeDataProcessAsync(sqlTransaction).ConfigureAwait(false); + + //And if initialized inside the Transaction then we also Clean up inside the Transaction! + if (sqlBulkHelpersConfig.MaterializedDataSchemaCopyMode == SchemaCopyMode.InsideTransactionAllowSchemaLocks) + { + await materializationDataHelper + .CleanupMaterializeDataProcessAsync(sqlTransaction, materializeDataContext) + .ConfigureAwait(false); + } + + //NOW we must commit our Transaction to save all Changes performed within the Transaction! + #if NETSTANDARD2_0 + sqlTransaction.Commit(); + #else + await sqlTransaction.CommitAsync().ConfigureAwait(false); + #endif + } + finally + { + //Some tasks MUST be handled outside of the Materialized Data Transaction (e.g. handling FullTextIndexes) + // so we handle those here (as needed and if enabled)... + await materializeDataContext.HandleParallelConnectionTasksAfterMaterialization().ConfigureAwait(false); + } + } + } + } + finally + { + //Initialize our Schema outside the transaction to avoid Schema locks (now Default Behaviour as of v2.3.0)... + if (materializeDataContext != null && sqlBulkHelpersConfig.MaterializedDataSchemaCopyMode == SchemaCopyMode.OutsideTransactionAvoidSchemaLocks) + { + #if NETSTANDARD2_0 + using (var sqlTransaction = sqlConnection.BeginTransaction()) + #else + await using (var sqlTransaction = (SqlTransaction)await sqlConnection.BeginTransactionAsync().ConfigureAwait(false)) + #endif + { + await materializationDataHelper + .CleanupMaterializeDataProcessAsync(sqlTransaction, materializeDataContext) + .ConfigureAwait(false); - //NOW we must commit our Transaction to save all Changes performed within the Transaction! #if NETSTANDARD2_0 sqlTransaction.Commit(); #else await sqlTransaction.CommitAsync().ConfigureAwait(false); #endif } - finally - { - //Some tasks MUST be handled outside of the Materialized Data Transaction (e.g. handling FullTextIndexes) - // so we handle those here (as needed and if enabled)... - await materializeDataContext.HandleNonTransactionTasksAfterMaterialization().ConfigureAwait(false); - } } } } diff --git a/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj b/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj index cc31278..6608e67 100644 --- a/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj +++ b/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj @@ -8,16 +8,20 @@ MIT BBernard / CajunCoding CajunCoding - 2.2.3 + 2.3.0 https://github.com/cajuncoding/SqlBulkHelpers https://github.com/cajuncoding/SqlBulkHelpers A library for easy, efficient and high performance bulk insert and update of data, into a Sql Database, from .Net applications. By leveraging the power of the SqlBulkCopy classes with added support for Identity primary key table columns this library provides a greatly simplified interface to process Identity based Entities with Bulk Performance with the wide compatibility of .NetStandard 2.0. sql server database table bulk insert update identity column sqlbulkcopy orm dapper linq2sql materialization materialized data view materialized-data materialized-view sync replication replica readonly + - Changed default behaviour to no longer clone tables/schema inside a Transaction which creates a full Schema Lock -- as this greatly impacts Schema aware ORMs such as SqlBulkHelpers, RepoDb, etc. + - New separate methods is now added to handle the CleanupMaterializeDataProcessAsync() but must be explicitly called as it is no longer implicitly called with FinishMaterializeDataProcessAsync(). + - Added new configuration value to control if Schema copying/cloning (for Loading Tables) is inside or outide the Transaction (e.g. SchemaCopyMode.InsideTransactionAllowSchemaLocks vs OutsideTransactionAvoidSchemaLocks). + - Fix bug in ReSeedTableIdentityValueWithMaxIdAsync() when the Table is Empty so that it now defaults to value of 1. + + Prior Relese Notes: - Fixed a Bug where Identity Column Value was not correctly synced after Materialization Process is completing. - Added new Helper API to quickly Sync the Identity column value with the current MAX Id value of the column (ensuring it's valid after populating additional data); This is useful if you override Identity values for a full Table refresh, but then want to later insert data into the table. - - Prior Relese Notes: - Improved namespace for SqlBulkHelpers.CustomExtensions to reduce risk of conflicts with similar existing extensions. - Restored support for SqlConnection Factory (simplified now as a Func<SqlConnection> when manually using the SqlDbSchemaLoader to dynamically retrieve Table Schema definitions for performance. - Added support for other Identity column data types including (INT, BIGINT, SMALLINT, & TINYINT); per feature request (https://github.com/cajuncoding/SqlBulkHelpers/issues/10). diff --git a/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs b/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs index 117a0ff..7e0cf95 100644 --- a/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs +++ b/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs @@ -10,6 +10,12 @@ public static class SqlBulkHelpersConfigConstants public const int DefaultMaxConcurrentConnections = 5; } + public enum SchemaCopyMode + { + OutsideTransactionAvoidSchemaLocks = 1, + InsideTransactionAllowSchemaLocks = 2, + } + public interface ISqlBulkHelpersConfig { int SqlBulkBatchSize { get; } //General guidance is that 2000-5000 is efficient enough. @@ -22,8 +28,17 @@ public interface ISqlBulkHelpersConfig int MaterializeDataStructureProcessingTimeoutSeconds { get; } int MaterializedDataSwitchTableWaitTimeoutMinutes { get; } SwitchWaitTimeoutAction MaterializedDataSwitchTimeoutAction { get; } + string MaterializedDataLoadingSchema { get; } + string MaterializedDataLoadingTablePrefix { get; } + string MaterializedDataLoadingTableSuffix { get; } + string MaterializedDataDiscardingSchema { get; } + string MaterializedDataDiscardingTablePrefix { get; } + string MaterializedDataDiscardingTableSuffix { get; } + + SchemaCopyMode MaterializedDataSchemaCopyMode { get; } + bool MaterializedDataMakeSchemaCopyNamesUnique { get; } bool IsCloningIdentitySeedValueEnabled { get; } ISqlBulkHelpersConnectionProvider ConcurrentConnectionFactory { get; } @@ -125,8 +140,17 @@ public bool IsSqlBulkTableLockEnabled public int MaterializeDataStructureProcessingTimeoutSeconds { get; set; } = 30; public int MaterializedDataSwitchTableWaitTimeoutMinutes { get; set; } = 1; public SwitchWaitTimeoutAction MaterializedDataSwitchTimeoutAction { get; } = SwitchWaitTimeoutAction.Abort; + public SchemaCopyMode MaterializedDataSchemaCopyMode { get; set; } = SchemaCopyMode.OutsideTransactionAvoidSchemaLocks; + public bool MaterializedDataMakeSchemaCopyNamesUnique { get; set; } = true; + public string MaterializedDataLoadingSchema { get; set; } = "materializing_load"; + public string MaterializedDataLoadingTablePrefix { get; set; } = string.Empty; + public string MaterializedDataLoadingTableSuffix { get; set; } = "_Loading"; + public string MaterializedDataDiscardingSchema { get; set; } = "materializing_discard"; + public string MaterializedDataDiscardingTablePrefix { get; set; } = string.Empty; + public string MaterializedDataDiscardingTableSuffix { get; set; } = "_Discarding"; + public bool IsCloningIdentitySeedValueEnabled { get; set; } = true; public ISqlBulkHelpersConnectionProvider ConcurrentConnectionFactory { get; set; } = null; diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs index feb0096..3408caf 100644 --- a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs +++ b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs @@ -31,7 +31,6 @@ public async Task TestCloneTableStructureByAnnotationAsync() Assert.IsNotNull(cloneInfo); Assert.AreEqual(TestHelpers.TestTableName, cloneInfo.SourceTable.TableName); Assert.AreNotEqual(cloneInfo.SourceTable.FullyQualifiedTableName, cloneInfo.TargetTable.FullyQualifiedTableName); - Assert.IsTrue(cloneInfo.TargetTable.TableName.Contains("_Copy_", StringComparison.OrdinalIgnoreCase)); //Validate the schema of the cloned table... Assert.IsNotNull(sourceTableSchema); @@ -87,6 +86,14 @@ public async Task TestCloneTableStructureIntoCustomTargetSchemaAsync() //Validate that the new table has No Data! var targetTableCount = await sqlConn.CountAllAsync(tableName: cloneInfo.TargetTable).ConfigureAwait(false); Assert.AreEqual(0, targetTableCount); + + //CLEANUP The Cloned Table so that other Tests Work as expected (e.g. Some tests validate Referencing FKeys, etc. + // that are now increased with the table clone). + await using (var sqlTransForCleanup = (SqlTransaction)await sqlConn.BeginTransactionAsync().ConfigureAwait(false)) + { + await sqlTransForCleanup.DropTableAsync(cloneInfo.TargetTable).ConfigureAwait(false); + await sqlTransForCleanup.CommitAsync().ConfigureAwait(false); + } } } @@ -121,6 +128,14 @@ public async Task TestCloneTableStructureWithCopiedDataAsync() //Ensure both Source & Target contain the same number of records! Assert.AreEqual(sourceTableCount, targetTableCount); + + //CLEANUP The Cloned Table so that other Tests Work as expected (e.g. Some tests validate Referencing FKeys, etc. + // that are now increased with the table clone). + await using (var sqlTransForCleanup = (SqlTransaction)await sqlConn.BeginTransactionAsync().ConfigureAwait(false)) + { + await sqlTransForCleanup.DropTableAsync(cloneInfo.TargetTable).ConfigureAwait(false); + await sqlTransForCleanup.CommitAsync().ConfigureAwait(false); + } } } diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs index 4be4392..ccc0311 100644 --- a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs +++ b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs @@ -4,7 +4,6 @@ using SqlBulkHelpers.MaterializedData; using Microsoft.Data.SqlClient; using RepoDb; -using SqlBulkHelpers.SqlBulkHelpers; using SqlBulkHelpers.Utilities; namespace SqlBulkHelpers.IntegrationTests @@ -177,6 +176,18 @@ public async Task TestMaterializeDataIntoTableWithIdentityColumnSyncingAsync() //FIRST CLEAR the Tables so we can validate that data changed (not coincidentally the same number of items)! await using (var sqlConn = await sqlConnectionProvider.NewConnectionAsync().ConfigureAwait(false)) { + await using (var sqlClearTableTransaction = (SqlTransaction)await sqlConn.BeginTransactionAsync().ConfigureAwait(false)) + { + await sqlClearTableTransaction.ClearTablesAsync(new[] + { + TestHelpers.TestTableNameFullyQualified, + TestHelpers.TestChildTableNameFullyQualified + }, + forceOverrideOfConstraints: true + ).ConfigureAwait(false); + await sqlClearTableTransaction.CommitAsync().ConfigureAwait(false); + } + var initialIdentityValue = await sqlConn.GetTableCurrentIdentityValueAsync(TestHelpers.TestTableNameFullyQualified).ConfigureAwait(false); int testDataCount = 15; @@ -324,7 +335,7 @@ public async Task TestMaterializeDataWithFKeyConstraintFailedValidationAsync() Exception sqlException = null; try { - await InsertDataWithInvalidFKeyState(validationEnabled: true); + await InsertDataWithInvalidFKeyStateAsync(validationEnabled: true); } catch (Exception exc) { @@ -342,7 +353,7 @@ public async Task TestMaterializeDataWithFKeyConstraintForcingOverrideOfValidati Exception sqlException = null; try { - await InsertDataWithInvalidFKeyState(validationEnabled: false); + await InsertDataWithInvalidFKeyStateAsync(validationEnabled: false); } catch (Exception exc) { @@ -354,54 +365,43 @@ public async Task TestMaterializeDataWithFKeyConstraintForcingOverrideOfValidati Assert.IsNull(sqlException); } - private async Task InsertDataWithInvalidFKeyState(bool validationEnabled) + private async Task InsertDataWithInvalidFKeyStateAsync(bool validationEnabled) { var sqlConnectionProvider = SqlConnectionHelper.GetConnectionProvider(); //NOW Materialize Data into the Tables! await using (var sqlConn = await sqlConnectionProvider.NewConnectionAsync().ConfigureAwait(false)) - await using (var sqlTrans = (SqlTransaction)await sqlConn.BeginTransactionAsync().ConfigureAwait(false)) { - ////Must clear all Data and Related Data to maintain Data Integrity... - ////NOTE: If we don't clear the related table then the FKey Constraint Check on the Related data (Child table) will FAIL! - //await sqlTrans.ClearTablesAsync(new[] - //{ - // TestHelpers.TestChildTableNameFullyQualified, - // TestHelpers.TestTableNameFullyQualified - //}, forceOverrideOfConstraints: true).ConfigureAwait(false); - - //****************************************************************************************** - //START the Materialize Data Process... - //****************************************************************************************** - var materializeDataContext = await sqlTrans.StartMaterializeDataProcessAsync( + var tableNames = new[] { TestHelpers.TestTableNameFullyQualified, TestHelpers.TestChildTableNameFullyQualified - ).ConfigureAwait(false); - - //Test with Table name being provided... - var parentMaterializationInfo = materializeDataContext[TestHelpers.TestTableName]; - var parentTestData = TestHelpers.CreateTestData(100); - var parentResults = (await sqlTrans.BulkInsertAsync(parentTestData, tableName: parentMaterializationInfo.LoadingTable).ConfigureAwait(false)).ToList(); - - //*********************************************************************** - //Now Clear the Parent Table to FORCE INVALID FKey STATE!!! - //*********************************************************************** - await sqlTrans.ClearTableAsync(parentMaterializationInfo.LoadingTable).ConfigureAwait(false); - - //Test Child Data with Table name being derived from Model Annotation... - var childMaterializationInfo = materializeDataContext[TestHelpers.TestChildTableNameFullyQualified]; - var childTestData = TestHelpers.CreateChildTestData(parentResults); - var childResults = await sqlTrans.BulkInsertOrUpdateAsync(childTestData, tableName: childMaterializationInfo.LoadingTable).ConfigureAwait(false); + }; - //By overriding this Value we force SQL Server to skip all validation of FKey constraints when re-enabling them! - // This can easily leave our data in an invalid state leaving the implementor responsible for ensuring Data Integrity of all tables being Materialized! - materializeDataContext.EnableDataConstraintChecksOnCompletion = validationEnabled; + await sqlConn.ExecuteMaterializeDataProcessAsync(tableNames, async (materializeDataContext, sqlTransaction) => + { + //Test with Table name being provided... + var parentMaterializationInfo = materializeDataContext[TestHelpers.TestTableName]; + var parentTestData = TestHelpers.CreateTestData(100); + var parentResults = (await sqlTransaction.BulkInsertAsync( + parentTestData, + tableName: parentMaterializationInfo.LoadingTable + ).ConfigureAwait(false)).ToList(); + + //*********************************************************************** + //Now Clear the Parent Table to FORCE INVALID FKey STATE!!! + //*********************************************************************** + await sqlTransaction.ClearTableAsync(parentMaterializationInfo.LoadingTable).ConfigureAwait(false); + + //Add Child Test Data with Table name being derived from Model Annotation... + var childMaterializationInfo = materializeDataContext[TestHelpers.TestChildTableNameFullyQualified]; + var childTestData = TestHelpers.CreateChildTestData(parentResults); + var childResults = await sqlTransaction.BulkInsertOrUpdateAsync(childTestData, tableName: childMaterializationInfo.LoadingTable).ConfigureAwait(false); + + //By overriding this Value we force SQL Server to skip all validation of FKey constraints when re-enabling them! + // This can easily leave our data in an invalid state leaving the implementor responsible for ensuring Data Integrity of all tables being Materialized! + materializeDataContext.EnableDataConstraintChecksOnCompletion = validationEnabled; - //****************************************************************************************** - //FINISH the Materialize Data Process... - //****************************************************************************************** - await materializeDataContext.FinishMaterializeDataProcessAsync().ConfigureAwait(false); - await sqlTrans.CommitAsync().ConfigureAwait(false); + }).ConfigureAwait(false); } } } diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/TableIdentityColumnApiTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/TableIdentityColumnApiTests.cs index 6f9e807..415ded1 100644 --- a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/TableIdentityColumnApiTests.cs +++ b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/TableIdentityColumnApiTests.cs @@ -109,7 +109,6 @@ public async Task TestReSeedTableIdentityValueWithMaxIdFromSqlTransactionSyncAnd var sqlConnectionString = SqlConnectionHelper.GetSqlConnectionString(); ISqlBulkHelpersConnectionProvider sqlConnectionProvider = new SqlBulkHelpersConnectionProvider(sqlConnectionString); - long initialIdentitySeedValue = 0; var firstNewIdentitySeedValue = 555888; var secondNewIdentitySeedValue = 888444;