From 572f725b887eb603e3af06f97a45c70236fcbdb4 Mon Sep 17 00:00:00 2001 From: Bruce Dunwiddie Date: Sat, 25 Jun 2022 00:40:34 -0500 Subject: [PATCH 1/2] Fixed parsing of CAST function arguments when data type specification include parenthesis. #98 #96 --- .../TSQLValueAsTypeExpressionParser.cs | 57 +++++++++++---- .../TSQL_Parser/Expressions/TSQLExpression.cs | 8 +++ .../Expressions/TSQLValueAsTypeExpression.cs | 2 +- TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj | 6 +- .../Tokens/Parsers/TSQLTokenFactory.cs | 8 +-- ...mmentToken.cs => TSQLIncompleteComment.cs} | 4 +- ...erToken.cs => TSQLIncompleteIdentifier.cs} | 4 +- ...StringToken.cs => TSQLIncompleteString.cs} | 4 +- .../Tests/Statements/SelectStatementTests.cs | 71 +++++++++++++------ .../Tests/Tokens/IncompleteTokenTests.cs | 10 +-- 10 files changed, 120 insertions(+), 54 deletions(-) rename TSQL_Parser/TSQL_Parser/Tokens/{TSQLIncompleteCommentToken.cs => TSQLIncompleteComment.cs} (74%) rename TSQL_Parser/TSQL_Parser/Tokens/{TSQLIncompleteIdentifierToken.cs => TSQLIncompleteIdentifier.cs} (73%) rename TSQL_Parser/TSQL_Parser/Tokens/{TSQLIncompleteStringToken.cs => TSQLIncompleteString.cs} (75%) diff --git a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueAsTypeExpressionParser.cs b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueAsTypeExpressionParser.cs index af8741b..3dfa00f 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueAsTypeExpressionParser.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueAsTypeExpressionParser.cs @@ -16,51 +16,78 @@ internal class TSQLValueAsTypeExpressionParser { public TSQLArgumentList Parse(ITSQLTokenizer tokenizer) { - List tokens = new List(); + TSQLValueAsTypeExpression argument = new TSQLValueAsTypeExpression(); // need to do this before starting the argument loop // so we can handle an empty argument list of just whitespace // and comments TSQLTokenParserHelper.ReadCommentsAndWhitespace( tokenizer, - tokens); - - TSQLValueAsTypeExpression argument = new TSQLValueAsTypeExpression(); + argument); TSQLExpression expression = new TSQLValueExpressionParser().Parse(tokenizer); argument.Expression = expression; - tokens.AddRange(expression.Tokens); + argument.Tokens.AddRange(expression.Tokens); TSQLTokenParserHelper.ReadCommentsAndWhitespace( tokenizer, - tokens); + argument); if (!tokenizer.Current.IsKeyword(TSQLKeywords.AS)) { throw new InvalidOperationException("AS expected."); } - tokens.Add(tokenizer.Current); + argument.Tokens.Add(tokenizer.Current); TSQLTokenParserHelper.ReadThroughAnyCommentsOrWhitespace( tokenizer, - tokens); + argument.Tokens); - argument.DataType = tokenizer.Current.AsIdentifier; + List dataTypeTokens = new List(); - tokens.Add(tokenizer.Current); + dataTypeTokens.Add(tokenizer.Current); - // reading until closing paren - TSQLTokenParserHelper.ReadThroughAnyCommentsOrWhitespace( - tokenizer, - tokens); + int nestedLevel = 0; + + while ( + tokenizer.MoveNext() && + !tokenizer.Current.IsCharacter(TSQLCharacters.Semicolon) && + !( + nestedLevel == 0 && + tokenizer.Current.IsCharacter(TSQLCharacters.CloseParentheses) + )) + { + if (tokenizer.Current.Type == TSQLTokenType.Character) + { + dataTypeTokens.Add(tokenizer.Current); + + TSQLCharacters character = tokenizer.Current.AsCharacter.Character; + + if (character == TSQLCharacters.OpenParentheses) + { + nestedLevel++; + } + else if (character == TSQLCharacters.CloseParentheses) + { + nestedLevel--; + } + } + else + { + dataTypeTokens.Add(tokenizer.Current); + } + } + + argument.DataType = String.Join("", dataTypeTokens.Select(t => t.Text)).TrimEnd(); + argument.Tokens.AddRange(dataTypeTokens); TSQLArgumentList argList = new TSQLArgumentList( new List { argument }); - argList.Tokens.AddRange(tokens); + argList.Tokens.AddRange(argument.Tokens); return argList; } diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs index c73fcf1..0b65bce 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs @@ -111,5 +111,13 @@ public TSQLVariableAssignmentExpression AsVariableAssignment return this as TSQLVariableAssignmentExpression; } } + + public TSQLValueAsTypeExpression AsValueAsType + { + get + { + return this as TSQLValueAsTypeExpression; + } + } } } diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLValueAsTypeExpression.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLValueAsTypeExpression.cs index 851c837..89662a6 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/TSQLValueAsTypeExpression.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLValueAsTypeExpression.cs @@ -20,6 +20,6 @@ public override TSQLExpressionType Type public TSQLExpression Expression { get; internal set; } - public TSQLIdentifier DataType { get; internal set; } + public string DataType { get; internal set; } } } diff --git a/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj b/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj index 4745879..4227243 100644 --- a/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj +++ b/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj @@ -174,9 +174,9 @@ - - - + + + diff --git a/TSQL_Parser/TSQL_Parser/Tokens/Parsers/TSQLTokenFactory.cs b/TSQL_Parser/TSQL_Parser/Tokens/Parsers/TSQLTokenFactory.cs index 9039921..92de801 100644 --- a/TSQL_Parser/TSQL_Parser/Tokens/Parsers/TSQLTokenFactory.cs +++ b/TSQL_Parser/TSQL_Parser/Tokens/Parsers/TSQLTokenFactory.cs @@ -60,7 +60,7 @@ public TSQLToken Parse( else { return - new TSQLIncompleteCommentToken( + new TSQLIncompleteComment( startPosition, tokenValue); } @@ -80,7 +80,7 @@ public TSQLToken Parse( else { return - new TSQLIncompleteStringToken( + new TSQLIncompleteString( startPosition, tokenValue); } @@ -100,7 +100,7 @@ public TSQLToken Parse( else { return - new TSQLIncompleteStringToken( + new TSQLIncompleteString( startPosition, tokenValue); } @@ -211,7 +211,7 @@ public TSQLToken Parse( )) { return - new TSQLIncompleteIdentifierToken( + new TSQLIncompleteIdentifier( startPosition, tokenValue); } diff --git a/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteCommentToken.cs b/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteComment.cs similarity index 74% rename from TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteCommentToken.cs rename to TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteComment.cs index 75cd0f9..c7b08ac 100644 --- a/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteCommentToken.cs +++ b/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteComment.cs @@ -2,9 +2,9 @@ namespace TSQL.Tokens { - public class TSQLIncompleteCommentToken : TSQLIncompleteToken + public class TSQLIncompleteComment : TSQLIncompleteToken { - internal TSQLIncompleteCommentToken( + internal TSQLIncompleteComment( int beginPosition, string text) : base( diff --git a/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteIdentifierToken.cs b/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteIdentifier.cs similarity index 73% rename from TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteIdentifierToken.cs rename to TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteIdentifier.cs index e61bb6a..31aedb6 100644 --- a/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteIdentifierToken.cs +++ b/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteIdentifier.cs @@ -2,9 +2,9 @@ namespace TSQL.Tokens { - public class TSQLIncompleteIdentifierToken : TSQLIncompleteToken + public class TSQLIncompleteIdentifier : TSQLIncompleteToken { - internal TSQLIncompleteIdentifierToken( + internal TSQLIncompleteIdentifier( int beginPosition, string text) : base( diff --git a/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteStringToken.cs b/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteString.cs similarity index 75% rename from TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteStringToken.cs rename to TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteString.cs index 8890821..5178907 100644 --- a/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteStringToken.cs +++ b/TSQL_Parser/TSQL_Parser/Tokens/TSQLIncompleteString.cs @@ -2,9 +2,9 @@ namespace TSQL.Tokens { - public class TSQLIncompleteStringToken : TSQLIncompleteToken + public class TSQLIncompleteString : TSQLIncompleteToken { - internal TSQLIncompleteStringToken( + internal TSQLIncompleteString( int beginPosition, string text) : base( diff --git a/TSQL_Parser/Tests/Statements/SelectStatementTests.cs b/TSQL_Parser/Tests/Statements/SelectStatementTests.cs index c4182cd..b624218 100644 --- a/TSQL_Parser/Tests/Statements/SelectStatementTests.cs +++ b/TSQL_Parser/Tests/Statements/SelectStatementTests.cs @@ -750,19 +750,9 @@ public void SelectStatement_system_user_Regression() { // regression test for https://github.com/bruce-dunwiddie/tsql-parser/issues/93 List statements = TSQLStatementReader.ParseStatements( - //@"SELECT system_user;", @"SELECT system_user;", includeWhitespace: false); - // System.NullReferenceException - // this shouldn't happen even if only because it encountered the end of the string - - // system_user is a system property, not a function - // is it trying to parse arguments? - - // should system properties be split out from system functions? - // what should each be named? - Assert.AreEqual(1, statements.Count); TSQLSelectStatement select = statements.Single().AsSelect; Assert.AreEqual(3, select.Tokens.Count); @@ -772,25 +762,66 @@ public void SelectStatement_system_user_Regression() [Test] public void SelectStatement_system_user_Regression_without_semicolon() { - // regression test for https://github.com/bruce-dunwiddie/tsql-parser/issues/93 List statements = TSQLStatementReader.ParseStatements( - //@"SELECT system_user;", @"SELECT system_user", includeWhitespace: false); - // System.NullReferenceException - // this shouldn't happen even if only because it encountered the end of the string + Assert.AreEqual(1, statements.Count); + TSQLSelectStatement select = statements.Single().AsSelect; + Assert.AreEqual(2, select.Tokens.Count); + Assert.AreEqual("system_user", select.Select.Columns[0].Expression.AsColumn.Column.Name); + } - // system_user is a system property, not a function - // is it trying to parse arguments? + [Test] + public void SelectStatement_CAST_argument_parsing() + { + // regression test for https://github.com/bruce-dunwiddie/tsql-parser/issues/98 + List statements = TSQLStatementReader.ParseStatements( + @"SELECT CAST(123.45 AS INT), CAST(456.321 AS VARCHAR(10)), Column_1 FROM MyTable", + includeWhitespace: false); - // should system properties be split out from system functions? - // what should each be named? + Assert.AreEqual(1, statements.Count); + TSQLSelectStatement select = statements.Single().AsSelect; + Assert.AreEqual(21, select.Tokens.Count); + Assert.AreEqual(3, select.Select.Columns.Count); + TSQLFunctionExpression function = select.Select.Columns[0].Expression.AsFunction; + Assert.AreEqual(6, function.Tokens.Count); + TSQLValueAsTypeExpression argument = function.Arguments[0].AsValueAsType; + Assert.AreEqual(3, argument.Tokens.Count); + Assert.AreEqual(123.45, argument.Expression.AsConstant.Literal.AsNumericLiteral.Value); + Assert.AreEqual("INT", argument.DataType); + function = select.Select.Columns[1].Expression.AsFunction; + Assert.AreEqual(9, function.Tokens.Count); + argument = function.Arguments[0].AsValueAsType; + Assert.AreEqual(6, argument.Tokens.Count); + Assert.AreEqual(456.321, argument.Expression.AsConstant.Literal.AsNumericLiteral.Value); + Assert.AreEqual("VARCHAR(10)", argument.DataType); + } + + [Test] + public void SelectStatement_CAST_argument_parsing_with_whitespace() + { + // regression test for https://github.com/bruce-dunwiddie/tsql-parser/issues/98 + List statements = TSQLStatementReader.ParseStatements( + @"SELECT CAST(123.45 AS INT), CAST(456.321 AS VARCHAR(10) ), Column_1 FROM MyTable", + includeWhitespace: true); Assert.AreEqual(1, statements.Count); TSQLSelectStatement select = statements.Single().AsSelect; - Assert.AreEqual(2, select.Tokens.Count); - Assert.AreEqual("system_user", select.Select.Columns[0].Expression.AsColumn.Column.Name); + Assert.AreEqual(31, select.Tokens.Count); + Assert.AreEqual(3, select.Select.Columns.Count); + TSQLFunctionExpression function = select.Select.Columns[0].Expression.AsFunction; + Assert.AreEqual(8, function.Tokens.Count); + TSQLValueAsTypeExpression argument = function.Arguments[0].AsValueAsType; + Assert.AreEqual(5, argument.Tokens.Count); + Assert.AreEqual(123.45, argument.Expression.AsConstant.Literal.AsNumericLiteral.Value); + Assert.AreEqual("INT", argument.DataType); + function = select.Select.Columns[1].Expression.AsFunction; + Assert.AreEqual(12, function.Tokens.Count); + argument = function.Arguments[0].AsValueAsType; + Assert.AreEqual(9, argument.Tokens.Count); + Assert.AreEqual(456.321, argument.Expression.AsConstant.Literal.AsNumericLiteral.Value); + Assert.AreEqual("VARCHAR(10)", argument.DataType); } } } diff --git a/TSQL_Parser/Tests/Tokens/IncompleteTokenTests.cs b/TSQL_Parser/Tests/Tokens/IncompleteTokenTests.cs index 666be87..c200726 100644 --- a/TSQL_Parser/Tests/Tokens/IncompleteTokenTests.cs +++ b/TSQL_Parser/Tests/Tokens/IncompleteTokenTests.cs @@ -21,7 +21,7 @@ public void IncompleteToken_StringLiteral() TokenComparisons.CompareTokenLists( new List() { - new TSQLIncompleteStringToken(0, "'") + new TSQLIncompleteString(0, "'") }, tokens); Assert.IsFalse(tokens[0].IsComplete); @@ -34,7 +34,7 @@ public void IncompleteToken_Identifier() TokenComparisons.CompareTokenLists( new List() { - new TSQLIncompleteIdentifierToken(0, "[dbo") + new TSQLIncompleteIdentifier(0, "[dbo") }, tokens); Assert.IsFalse(tokens[0].IsComplete); @@ -47,7 +47,7 @@ public void IncompleteToken_Comment() TokenComparisons.CompareTokenLists( new List() { - new TSQLIncompleteCommentToken(0, "/* something") + new TSQLIncompleteComment(0, "/* something") }, tokens); Assert.IsFalse(tokens[0].IsComplete); @@ -63,7 +63,7 @@ public void IncompleteToken_DoubleQuoteIdentifier() TokenComparisons.CompareTokenLists( new List() { - new TSQLIncompleteIdentifierToken(0, "\"dbo") + new TSQLIncompleteIdentifier(0, "\"dbo") }, tokens); Assert.IsFalse(tokens[0].IsComplete); @@ -79,7 +79,7 @@ public void IncompleteToken_DoubleQuoteString() TokenComparisons.CompareTokenLists( new List() { - new TSQLIncompleteStringToken(0, "\"something") + new TSQLIncompleteString(0, "\"something") }, tokens); Assert.IsFalse(tokens[0].IsComplete); From ceda5cf4075b7e7d504ba91a58e87f08cfcfadf7 Mon Sep 17 00:00:00 2001 From: Bruce Dunwiddie Date: Sun, 26 Jun 2022 21:45:36 -0500 Subject: [PATCH 2/2] Updated version numbers for release. --- TSQL_Parser/TSQL_Parser/Properties/AssemblyInfo.cs | 4 ++-- TSQL_Parser/TSQL_Parser/Push.bat | 4 ++-- TSQL_Parser/TSQL_Parser/TSQL_Parser.nuspec | 4 ++-- TSQL_Parser/TSQL_Parser/TSQL_Parser_NetStandard.csproj | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/TSQL_Parser/TSQL_Parser/Properties/AssemblyInfo.cs b/TSQL_Parser/TSQL_Parser/Properties/AssemblyInfo.cs index 505e556..8027805 100644 --- a/TSQL_Parser/TSQL_Parser/Properties/AssemblyInfo.cs +++ b/TSQL_Parser/TSQL_Parser/Properties/AssemblyInfo.cs @@ -32,7 +32,7 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.2.1.0")] -[assembly: AssemblyFileVersion("2.2.1.0")] +[assembly: AssemblyVersion("2.2.2.0")] +[assembly: AssemblyFileVersion("2.2.2.0")] [assembly: InternalsVisibleTo("Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100793625650b945744f8a2c57bc75da89cd4d2c551636aa180c3020b7a15b815c10e983e83c312eb02f131c6fcf18aaffd6c8d9af6c4353c91ca0e9206b0fb8fb7805fc07b510a47ff40705ae56977ae8893e2d247d166aa400926582840e8a5602df055762bc3479dd14c9621a77946b6e6b0a00a77204c78fb52c65121bd75ba")] \ No newline at end of file diff --git a/TSQL_Parser/TSQL_Parser/Push.bat b/TSQL_Parser/TSQL_Parser/Push.bat index 738606d..8bc9b1b 100644 --- a/TSQL_Parser/TSQL_Parser/Push.bat +++ b/TSQL_Parser/TSQL_Parser/Push.bat @@ -1,4 +1,4 @@ nuget SetApiKey %NUGET_KEY% -nuget push TSQL.Parser.2.2.1.snupkg -Source https://api.nuget.org/v3/index.json -nuget push TSQL.Parser.2.2.1.nupkg -Source https://api.nuget.org/v3/index.json +nuget push TSQL.Parser.2.2.2.snupkg -Source https://api.nuget.org/v3/index.json +nuget push TSQL.Parser.2.2.2.nupkg -Source https://api.nuget.org/v3/index.json pause \ No newline at end of file diff --git a/TSQL_Parser/TSQL_Parser/TSQL_Parser.nuspec b/TSQL_Parser/TSQL_Parser/TSQL_Parser.nuspec index 5ff2d80..91eff84 100644 --- a/TSQL_Parser/TSQL_Parser/TSQL_Parser.nuspec +++ b/TSQL_Parser/TSQL_Parser/TSQL_Parser.nuspec @@ -2,7 +2,7 @@ TSQL.Parser - 2.2.1 + 2.2.2 TSQL.Parser Bruce Dunwiddie shriop @@ -10,7 +10,7 @@ https://github.com/bruce-dunwiddie/tsql-parser false Library for Parsing SQL Server T-SQL Scripts - Fixed simple single column SELECT parsing. + Fixed parsing of CAST function arguments when data type specification includes parentheses. Copyright © 2022 sql parser sql-server tsql diff --git a/TSQL_Parser/TSQL_Parser/TSQL_Parser_NetStandard.csproj b/TSQL_Parser/TSQL_Parser/TSQL_Parser_NetStandard.csproj index d03abf5..a11e502 100644 --- a/TSQL_Parser/TSQL_Parser/TSQL_Parser_NetStandard.csproj +++ b/TSQL_Parser/TSQL_Parser/TSQL_Parser_NetStandard.csproj @@ -4,9 +4,9 @@ netstandard2.0 TSQL_Parser TSQL_Parser - 2.2.1.0 - 2.2.1.0 - 2.2.1.0 + 2.2.2.0 + 2.2.2.0 + 2.2.2.0 Library for Parsing SQL Server TSQL Scripts Copyright © 2022