From 991ba578ca5c0911dd3754e956fa2d454e959b9b Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Thu, 18 Apr 2024 12:31:54 -0600 Subject: [PATCH 01/16] custom parameters all seem to be working, still have a few more tests to write --- src/defines.ts | 11 ++ src/index.ts | 2 +- src/parser.ts | 4 +- src/tokenizer.ts | 123 +++++++++++++++++++- test/tokenizer/index.spec.ts | 213 ++++++++++++++++++++++++++++++++++- 5 files changed, 344 insertions(+), 9 deletions(-) diff --git a/src/defines.ts b/src/defines.ts index 955d04f..3d0721f 100644 --- a/src/defines.ts +++ b/src/defines.ts @@ -76,10 +76,21 @@ export type StatementType = export type ExecutionType = 'LISTING' | 'MODIFICATION' | 'INFORMATION' | 'ANON_BLOCK' | 'UNKNOWN'; +export interface ParamTypes { + positional?: boolean, + numbered?: Array<"?" | ":" | "$">, + named?: Array<":" | "@" | "$">, + quoted?: Array<":" | "@" | "$">, + // regex is for identifying that it is a param, key is how the token is translated to an object value for the formatter, + // may not be necessary here, we shal see + custom?: Array<{regex: string, key?: (text: string) => string }> +} + export interface IdentifyOptions { strict?: boolean; dialect?: Dialect; identifyTables?: boolean; + paramTypes?: ParamTypes } export interface IdentifyResult { diff --git a/src/index.ts b/src/index.ts index c57dafd..a47599e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,7 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify throw new Error(`Unknown dialect. Allowed values: ${DIALECTS.join(', ')}`); } - const result = parse(query, isStrict, dialect, options.identifyTables); + const result = parse(query, isStrict, dialect, options.identifyTables, options.paramTypes); return result.body.map((statement) => { const result: IdentifyResult = { diff --git a/src/parser.ts b/src/parser.ts index 3aec638..d408260 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -9,6 +9,7 @@ import type { Step, ParseResult, ConcreteStatement, + ParamTypes, } from './defines'; interface StatementParser { @@ -144,6 +145,7 @@ export function parse( isStrict = true, dialect: Dialect = 'generic', identifyTables = false, + paramTypes?: ParamTypes ): ParseResult { const topLevelState = initState({ input }); const topLevelStatement: ParseResult = { @@ -174,7 +176,7 @@ export function parse( while (prevState.position < topLevelState.end) { const tokenState = initState({ prevState }); - const token = scanToken(tokenState, dialect); + const token = scanToken(tokenState, dialect, paramTypes); const nextToken = nextNonWhitespaceToken(tokenState, dialect); if (!statementParser) { diff --git a/src/tokenizer.ts b/src/tokenizer.ts index eccfbc7..4f46505 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -2,7 +2,7 @@ * Tokenizer */ -import type { Token, State, Dialect } from './defines'; +import type { Token, State, Dialect, ParamTypes } from './defines'; type Char = string | null; @@ -76,7 +76,7 @@ const ENDTOKENS: Record = { '[': ']', }; -export function scanToken(state: State, dialect: Dialect = 'generic'): Token { +export function scanToken(state: State, dialect: Dialect = 'generic', paramTypes?: ParamTypes): Token { const ch = read(state); if (isWhitespace(ch)) { @@ -95,8 +95,8 @@ export function scanToken(state: State, dialect: Dialect = 'generic'): Token { return scanString(state, ENDTOKENS[ch]); } - if (isParameter(ch, state, dialect)) { - return scanParameter(state, dialect); + if (isParameter(ch, state, dialect, paramTypes)) { + return scanParameter(state, dialect, paramTypes); } if (isDollarQuotedString(state)) { @@ -253,7 +253,88 @@ function scanString(state: State, endToken: Char): Token { }; } -function scanParameter(state: State, dialect: Dialect): Token { +function getCustomParam(state: State, paramTypes: ParamTypes): string | null | undefined { + const matches = paramTypes?.custom?.map(({ regex }) => { + const reg = new RegExp(`(?:${regex})`, 'u'); + return reg.exec(state.input); + }).filter((value) => !!value)[0]; + + return matches ? matches[0] : null; +} + +function scanParameter(state: State, dialect: Dialect, paramTypes?: ParamTypes): Token { + // user has defined wanted param types, so we only evaluate them + if (paramTypes) { + const curCh: any = state.input[0]; + let nextChar = peek(state); + let matched = false + + // this could be a named parameter that just starts with a number (ugh) + if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { + const maybeNumbers = state.input.slice(1, state.input.length); + if (nextChar !== null && !isNaN(Number(nextChar)) && /^\d+$/.test(maybeNumbers)) { + do { + nextChar = read(state); + } while (nextChar !== null && !isNaN(Number(nextChar)) && !isWhitespace(nextChar)); + + if (nextChar !== null) unread(state); + matched = true; + } + } + + if (!matched && paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) { + if (!isQuotedIdentifier(nextChar, dialect)) { + while (isAlphaNumeric(peek(state))) read(state); + matched = true; + } + } + + if (!matched && paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) { + if (isQuotedIdentifier(nextChar, dialect)) { + const endChars = new Map([ + ['"', '"'], + ['[', ']'], + ['`', '`'] + ]); + const quoteChar = read(state) as string; + const end = endChars.get(quoteChar); + // end when we reach the end quote + while ((isAlphaNumeric(peek(state)) || peek(state) === ' ') && peek(state) != end) read(state); + + // read the end quote + read(state); + + matched = true; + } + } + + if (!matched && paramTypes.custom && paramTypes.custom.length) { + const custom = getCustomParam(state, paramTypes); + + if (custom) { + read(state, custom.length); + matched = true; + } + } + + if (!matched && curCh !== '?' && nextChar !== null) { // not positional, panic + return { + type: 'parameter', + value: 'unknown', + start: state.start, + end: state.end + } + } + + const value = state.input.slice(state.start, state.position + 1); + return { + type: 'parameter', + value, + start: state.start, + end: state.start + value.length - 1, + }; + } + if (['mysql', 'generic', 'sqlite'].includes(dialect)) { return { type: 'parameter', @@ -413,7 +494,37 @@ function isString(ch: Char, dialect: Dialect): boolean { return stringStart.includes(ch); } -function isParameter(ch: Char, state: State, dialect: Dialect): boolean { +function isCustomParam(state: State, paramTypes: ParamTypes): boolean | undefined { + return paramTypes?.custom?.some(({ regex }) => { + const reg = new RegExp(`(?:${regex})`, 'uy'); + return reg.test(state.input); + }) +} + +function isParameter(ch: Char, state: State, dialect: Dialect, paramTypes?: ParamTypes): boolean { + if (paramTypes && ch !== null) { + const curCh: any = ch; + const nextChar = peek(state); + if (paramTypes.positional && ch === '?' && nextChar === null) return true; + + if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { + if (nextChar !== null && !isNaN(Number(nextChar))) { + return true; + } + } + + if ((paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) || + (paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh))) { + return true; + } + + if ((paramTypes.custom && paramTypes.custom.length && isCustomParam(state, paramTypes))) { + return true + } + + return false; + } + let pStart = '?'; // ansi standard - sqlite, mysql if (dialect === 'psql') { pStart = '$'; diff --git a/test/tokenizer/index.spec.ts b/test/tokenizer/index.spec.ts index 9d16f34..a541599 100644 --- a/test/tokenizer/index.spec.ts +++ b/test/tokenizer/index.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; import { scanToken } from '../../src/tokenizer'; -import type { Dialect } from '../../src/defines'; +import type { Dialect, ParamTypes } from '../../src/defines'; describe('scan', () => { const initState = (input: string) => ({ @@ -376,6 +376,217 @@ describe('scan', () => { }, ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); }); + + describe('custom parameters', () => { + it('should allow positional parameters for all dialects', () => { + const paramTypes: ParamTypes = { + positional: true + }; + + const expected = { + type: 'parameter', + value: '?', + start: 0, + end: 0 + }; + + + (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + [ + { + actual: scanToken(initState('?'), dialect, paramTypes), + expected, + }, + ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + }); + }); + + it('should allow numeric parameters for all dialects', () => { + const paramTypes: ParamTypes = { + numbered: ["$", "?", ":"] + }; + + const expected = [ + { + type: 'parameter', + value: '$1', + start: 0, + end: 1 + }, + { + type: 'parameter', + value: '?1', + start: 0, + end: 1 + }, + { + type: 'parameter', + value: ':1', + start: 0, + end: 1 + }, + { + type: 'parameter', + value: 'unknown', + start: 0, + end: 8 + } + ]; + + (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + [ + { + actual: scanToken(initState('$1'), dialect, paramTypes), + expected: expected[0], + }, + { + actual: scanToken(initState('?1'), dialect, paramTypes), + expected: expected[1], + }, + { + actual: scanToken(initState(':1'), dialect, paramTypes), + expected: expected[2], + }, + { + actual: scanToken(initState('$123hello'), dialect, paramTypes), // won't recognize + expected: expected[3] + } + ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + }); + }); + + it('should allow named parameters for all dialects', () => { + const paramTypes: ParamTypes = { + named: ["$", "@", ":"] + }; + + const expected = [ + { + type: 'parameter', + value: '$namedParam', + start: 0, + end: 10 + }, + { + type: 'parameter', + value: '@namedParam', + start: 0, + end: 10 + }, + { + type: 'parameter', + value: ':namedParam', + start: 0, + end: 10 + }, + { + type: 'parameter', + value: '$123hello', // allow starting with a number + start: 0, + end: 8 + } + ]; + + (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + [ + { + actual: scanToken(initState('$namedParam'), dialect, paramTypes), + expected: expected[0], + }, + { + actual: scanToken(initState('@namedParam'), dialect, paramTypes), + expected: expected[1], + }, + { + actual: scanToken(initState(':namedParam'), dialect, paramTypes), + expected: expected[2], + }, + { + actual: scanToken(initState('$123hello'), dialect, paramTypes), + expected: expected[3] + } + ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + }) + }); + + // this test will need a refactor depending on how we want to implement quotes + it('should allow quoted parameters for all dialects', () => { + const paramTypes: ParamTypes = { + quoted: ["$", "@", ":"] + }; + + const expected = [ + { + type: 'parameter', + value: '$', + start: 0, + end: 14 + }, + { + type: 'parameter', + value: '@', + start: 0, + end: 14 + }, + { + type: 'parameter', + value: ':', + start: 0, + end: 14 + } + ]; + + ([ + { dialect: 'mssql', quotes: ['""', '[]'] }, + { dialect: 'psql', quotes: ['""', '``'] }, + { dialect: 'oracle', quotes: ['""', '``']}, + { dialect: 'bigquery', quotes: ['""', '``']}, + { dialect: 'sqlite', quotes: ['""', '``']}, + { dialect: 'mysql', quotes: ['""', '``']}, + { dialect: 'generic', quotes: ['""', '``']}, + ] as Array<{dialect: Dialect, quotes: Array}>).forEach(({dialect, quotes}) => { + const dialectExpected = expected.map((exp) => { + return quotes.map((quote) => { + return { + ...exp, + value: `${exp.value}${quote[0]}quoted param${quote[1]}` + } + }) + }).flat(); + dialectExpected.map((expected) => ({ + actual: scanToken(initState(expected.value), dialect, paramTypes), + expected + })).forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + }) + }); + + it('should allow custom parameters for all dialects', () => { + const paramTypes: ParamTypes = { + custom: [{ regex: '\\{[a-zA-Z0-9_]+\\}' }] + }; + + const expected = { + type: 'parameter', + value: '{namedParam}', + start: 0, + end: 11 + }; + + (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + expect(scanToken(initState('{namedParam}'), dialect, paramTypes)).to.eql(expected); + }) + }); + + it('should not have collision between param types', () => { + const paramTypes: ParamTypes = { + positional: true, + numbered: [':'], + named: [':'], + quoted: [':'], + custom: [] + }; + }) + }); }); }); }); From 41c17d76e033afa48085138bb44f1b6d0501b106 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Thu, 18 Apr 2024 15:19:00 -0600 Subject: [PATCH 02/16] fix numeric and custom tokens --- src/defines.ts | 5 +- src/index.ts | 3 +- src/tokenizer.ts | 144 +++++++++++++++++------------------ test/index.spec.ts | 24 ++++++ test/tokenizer/index.spec.ts | 33 +++++++- 5 files changed, 132 insertions(+), 77 deletions(-) diff --git a/src/defines.ts b/src/defines.ts index 3d0721f..e4b4b1f 100644 --- a/src/defines.ts +++ b/src/defines.ts @@ -81,9 +81,8 @@ export interface ParamTypes { numbered?: Array<"?" | ":" | "$">, named?: Array<":" | "@" | "$">, quoted?: Array<":" | "@" | "$">, - // regex is for identifying that it is a param, key is how the token is translated to an object value for the formatter, - // may not be necessary here, we shal see - custom?: Array<{regex: string, key?: (text: string) => string }> + // regex for identifying that it is a param + custom?: Array } export interface IdentifyOptions { diff --git a/src/index.ts b/src/index.ts index a47599e..fb34955 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify } const result = parse(query, isStrict, dialect, options.identifyTables, options.paramTypes); + const sort = dialect === 'psql' && !options.paramTypes; return result.body.map((statement) => { const result: IdentifyResult = { @@ -31,7 +32,7 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify type: statement.type, executionType: statement.executionType, // we want to sort the postgres params: $1 $2 $3, regardless of the order they appear - parameters: dialect === 'psql' ? statement.parameters.sort() : statement.parameters, + parameters: sort ? statement.parameters.sort() : statement.parameters, tables: statement.tables || [], }; return result; diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 4f46505..fc816ba 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -254,85 +254,85 @@ function scanString(state: State, endToken: Char): Token { } function getCustomParam(state: State, paramTypes: ParamTypes): string | null | undefined { - const matches = paramTypes?.custom?.map(({ regex }) => { - const reg = new RegExp(`(?:${regex})`, 'u'); - return reg.exec(state.input); + const matches = paramTypes?.custom?.map((regex) => { + const reg = new RegExp(`^(?:${regex})`, 'u'); + return reg.exec(state.input.slice(state.start)); }).filter((value) => !!value)[0]; return matches ? matches[0] : null; } -function scanParameter(state: State, dialect: Dialect, paramTypes?: ParamTypes): Token { - // user has defined wanted param types, so we only evaluate them - if (paramTypes) { - const curCh: any = state.input[0]; - let nextChar = peek(state); - let matched = false +function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): Token { - // this could be a named parameter that just starts with a number (ugh) - if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { - const maybeNumbers = state.input.slice(1, state.input.length); - if (nextChar !== null && !isNaN(Number(nextChar)) && /^\d+$/.test(maybeNumbers)) { - do { - nextChar = read(state); - } while (nextChar !== null && !isNaN(Number(nextChar)) && !isWhitespace(nextChar)); - - if (nextChar !== null) unread(state); - matched = true; - } - } - - if (!matched && paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) { - if (!isQuotedIdentifier(nextChar, dialect)) { - while (isAlphaNumeric(peek(state))) read(state); - matched = true; - } - } - - if (!matched && paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) { - if (isQuotedIdentifier(nextChar, dialect)) { - const endChars = new Map([ - ['"', '"'], - ['[', ']'], - ['`', '`'] - ]); - const quoteChar = read(state) as string; - const end = endChars.get(quoteChar); - // end when we reach the end quote - while ((isAlphaNumeric(peek(state)) || peek(state) === ' ') && peek(state) != end) read(state); - - // read the end quote - read(state); - - matched = true; - } - } - - if (!matched && paramTypes.custom && paramTypes.custom.length) { - const custom = getCustomParam(state, paramTypes); + const curCh: any = state.input[state.start]; + let nextChar = peek(state); + let matched = false - if (custom) { - read(state, custom.length); - matched = true; - } - } - - if (!matched && curCh !== '?' && nextChar !== null) { // not positional, panic - return { - type: 'parameter', - value: 'unknown', - start: state.start, - end: state.end - } - } + if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { + const endIndex = state.input.slice(state.start).split('').findIndex((val) => isWhitespace(val)); + const maybeNumbers = state.input.slice(state.start + 1, endIndex > 0 ? state.start + endIndex : state.end + 1); + if (nextChar !== null && !isNaN(Number(nextChar)) && /^\d+$/.test(maybeNumbers)) { + let nextChar: Char = null; + do { + nextChar = read(state); + } while (nextChar !== null && !isNaN(Number(nextChar)) && !isWhitespace(nextChar)); - const value = state.input.slice(state.start, state.position + 1); + if (nextChar !== null) unread(state); + matched = true; + } + } + + if (!matched && paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) { + if (!isQuotedIdentifier(nextChar, dialect)) { + while (isAlphaNumeric(peek(state))) read(state); + matched = true; + } + } + + if (!matched && paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) { + if (isQuotedIdentifier(nextChar, dialect)) { + const quoteChar = read(state) as string; + // end when we reach the end quote + while ((isAlphaNumeric(peek(state)) || peek(state) === ' ') && peek(state) != ENDTOKENS[quoteChar]) read(state); + + // read the end quote + read(state); + + matched = true; + } + } + + if (!matched && paramTypes.custom && paramTypes.custom.length) { + const custom = getCustomParam(state, paramTypes); + + if (custom) { + read(state, custom.length); + matched = true; + } + } + + if (!matched && !paramTypes.positional) { // not positional, panic return { type: 'parameter', - value, + value: 'unknown', start: state.start, - end: state.start + value.length - 1, - }; + end: state.end + } + } + + const value = state.input.slice(state.start, state.position + 1); + return { + type: 'parameter', + value, + start: state.start, + end: state.start + value.length - 1, + }; +} + +function scanParameter(state: State, dialect: Dialect, paramTypes?: ParamTypes): Token { + // user has defined wanted param types, so we only evaluate them + if (paramTypes) { + return scanCustomParameter(state, dialect, paramTypes); } if (['mysql', 'generic', 'sqlite'].includes(dialect)) { @@ -495,9 +495,9 @@ function isString(ch: Char, dialect: Dialect): boolean { } function isCustomParam(state: State, paramTypes: ParamTypes): boolean | undefined { - return paramTypes?.custom?.some(({ regex }) => { - const reg = new RegExp(`(?:${regex})`, 'uy'); - return reg.test(state.input); + return paramTypes?.custom?.some((regex) => { + const reg = new RegExp(`^(?:${regex})`, 'uy'); + return reg.test(state.input.slice(state.start)); }) } @@ -505,7 +505,7 @@ function isParameter(ch: Char, state: State, dialect: Dialect, paramTypes?: Para if (paramTypes && ch !== null) { const curCh: any = ch; const nextChar = peek(state); - if (paramTypes.positional && ch === '?' && nextChar === null) return true; + if (paramTypes.positional && ch === '?' && (nextChar === null || isWhitespace(nextChar))) return true; if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { if (nextChar !== null && !isNaN(Number(nextChar))) { diff --git a/test/index.spec.ts b/test/index.spec.ts index d7cd5e9..200a504 100644 --- a/test/index.spec.ts +++ b/test/index.spec.ts @@ -1,5 +1,6 @@ import { Dialect, getExecutionType, identify } from '../src/index'; import { expect } from 'chai'; +import { ParamTypes } from '../src/defines'; describe('identify', () => { it('should throw error for invalid dialect', () => { @@ -22,6 +23,29 @@ describe('identify', () => { ]); }); + it('should identify custom parameters', () => { + const paramTypes: ParamTypes = { + positional: true, + numbered: ['$'], + named: [':'], + quoted: [':'], + custom: [ '\\{[a-zA-Z0-9_]+\\}' ] + }; + const query = `SELECT * FROM foo WHERE bar = ? AND baz = $1 AND fizz = :fizzz AND buzz = :"buzz buzz" AND foo2 = {fooo}`; + + expect(identify(query, { dialect: 'psql', paramTypes })).to.eql([ + { + start: 0, + end: 104, + text: query, + type: 'SELECT', + executionType: 'LISTING', + parameters: ['?', '$1', ':fizzz', ':"buzz buzz"', '{fooo}'], + tables: [] + } + ]) + }) + it('should identify tables in simple for basic cases', () => { expect( identify('SELECT * FROM foo JOIN bar ON foo.id = bar.id', { identifyTables: true }), diff --git a/test/tokenizer/index.spec.ts b/test/tokenizer/index.spec.ts index a541599..89e4f4c 100644 --- a/test/tokenizer/index.spec.ts +++ b/test/tokenizer/index.spec.ts @@ -562,7 +562,7 @@ describe('scan', () => { it('should allow custom parameters for all dialects', () => { const paramTypes: ParamTypes = { - custom: [{ regex: '\\{[a-zA-Z0-9_]+\\}' }] + custom: [ '\\{[a-zA-Z0-9_]+\\}' ] }; const expected = { @@ -585,6 +585,37 @@ describe('scan', () => { quoted: [':'], custom: [] }; + + const expected = [ + { + type: 'parameter', + value: '?', + start: 0, + end: 0 + }, + { + type: 'parameter', + value: ':123', + start: 0, + end: 3 + }, + { + type: 'parameter', + value: ':123hello', + start: 0, + end: 8 + }, + { + type: 'parameter', + value: ':"named param"', + start: 0, + end: 13 + } + ]; + + expected.forEach((expected) => { + expect(scanToken(initState(expected.value), 'mssql', paramTypes)).to.eql(expected); + }) }) }); }); From 300c1e1d03dcc6bae5f430a3109c52ce6c3172e0 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Thu, 18 Apr 2024 16:27:53 -0600 Subject: [PATCH 03/16] fix linting and test for old node --- src/defines.ts | 12 +-- src/parser.ts | 2 +- src/tokenizer.ts | 84 +++++++++++++-------- test/index.spec.ts | 10 +-- test/tokenizer/index.spec.ts | 142 +++++++++++++++++++---------------- 5 files changed, 144 insertions(+), 106 deletions(-) diff --git a/src/defines.ts b/src/defines.ts index e4b4b1f..012331b 100644 --- a/src/defines.ts +++ b/src/defines.ts @@ -77,19 +77,19 @@ export type StatementType = export type ExecutionType = 'LISTING' | 'MODIFICATION' | 'INFORMATION' | 'ANON_BLOCK' | 'UNKNOWN'; export interface ParamTypes { - positional?: boolean, - numbered?: Array<"?" | ":" | "$">, - named?: Array<":" | "@" | "$">, - quoted?: Array<":" | "@" | "$">, + positional?: boolean; + numbered?: Array<'?' | ':' | '$'>; + named?: Array<':' | '@' | '$'>; + quoted?: Array<':' | '@' | '$'>; // regex for identifying that it is a param - custom?: Array + custom?: Array; } export interface IdentifyOptions { strict?: boolean; dialect?: Dialect; identifyTables?: boolean; - paramTypes?: ParamTypes + paramTypes?: ParamTypes; } export interface IdentifyResult { diff --git a/src/parser.ts b/src/parser.ts index d408260..57bbff5 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -145,7 +145,7 @@ export function parse( isStrict = true, dialect: Dialect = 'generic', identifyTables = false, - paramTypes?: ParamTypes + paramTypes?: ParamTypes, ): ParseResult { const topLevelState = initState({ input }); const topLevelStatement: ParseResult = { diff --git a/src/tokenizer.ts b/src/tokenizer.ts index fc816ba..94ddfa6 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -76,7 +76,11 @@ const ENDTOKENS: Record = { '[': ']', }; -export function scanToken(state: State, dialect: Dialect = 'generic', paramTypes?: ParamTypes): Token { +export function scanToken( + state: State, + dialect: Dialect = 'generic', + paramTypes?: ParamTypes, +): Token { const ch = read(state); if (isWhitespace(ch)) { @@ -254,23 +258,30 @@ function scanString(state: State, endToken: Char): Token { } function getCustomParam(state: State, paramTypes: ParamTypes): string | null | undefined { - const matches = paramTypes?.custom?.map((regex) => { - const reg = new RegExp(`^(?:${regex})`, 'u'); - return reg.exec(state.input.slice(state.start)); - }).filter((value) => !!value)[0]; + const matches = paramTypes?.custom + ?.map((regex) => { + const reg = new RegExp(`^(?:${regex})`, 'u'); + return reg.exec(state.input.slice(state.start)); + }) + .filter((value) => !!value)[0]; return matches ? matches[0] : null; } function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): Token { - const curCh: any = state.input[state.start]; - let nextChar = peek(state); - let matched = false + const nextChar = peek(state); + let matched = false; if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { - const endIndex = state.input.slice(state.start).split('').findIndex((val) => isWhitespace(val)); - const maybeNumbers = state.input.slice(state.start + 1, endIndex > 0 ? state.start + endIndex : state.end + 1); + const endIndex = state.input + .slice(state.start) + .split('') + .findIndex((val) => isWhitespace(val)); + const maybeNumbers = state.input.slice( + state.start + 1, + endIndex > 0 ? state.start + endIndex : state.end + 1, + ); if (nextChar !== null && !isNaN(Number(nextChar)) && /^\d+$/.test(maybeNumbers)) { let nextChar: Char = null; do { @@ -280,28 +291,37 @@ function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTy if (nextChar !== null) unread(state); matched = true; } - } - + } + if (!matched && paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) { if (!isQuotedIdentifier(nextChar, dialect)) { while (isAlphaNumeric(peek(state))) read(state); matched = true; } - } - - if (!matched && paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) { + } + + if ( + !matched && + paramTypes.quoted && + paramTypes.quoted.length && + paramTypes.quoted.includes(curCh) + ) { if (isQuotedIdentifier(nextChar, dialect)) { const quoteChar = read(state) as string; // end when we reach the end quote - while ((isAlphaNumeric(peek(state)) || peek(state) === ' ') && peek(state) != ENDTOKENS[quoteChar]) read(state); + while ( + (isAlphaNumeric(peek(state)) || peek(state) === ' ') && + peek(state) != ENDTOKENS[quoteChar] + ) + read(state); // read the end quote read(state); matched = true; } - } - + } + if (!matched && paramTypes.custom && paramTypes.custom.length) { const custom = getCustomParam(state, paramTypes); @@ -309,15 +329,16 @@ function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTy read(state, custom.length); matched = true; } - } - - if (!matched && !paramTypes.positional) { // not positional, panic + } + + if (!matched && !paramTypes.positional) { + // not positional, panic return { type: 'parameter', value: 'unknown', start: state.start, - end: state.end - } + end: state.end, + }; } const value = state.input.slice(state.start, state.position + 1); @@ -498,14 +519,15 @@ function isCustomParam(state: State, paramTypes: ParamTypes): boolean | undefine return paramTypes?.custom?.some((regex) => { const reg = new RegExp(`^(?:${regex})`, 'uy'); return reg.test(state.input.slice(state.start)); - }) + }); } function isParameter(ch: Char, state: State, dialect: Dialect, paramTypes?: ParamTypes): boolean { if (paramTypes && ch !== null) { const curCh: any = ch; const nextChar = peek(state); - if (paramTypes.positional && ch === '?' && (nextChar === null || isWhitespace(nextChar))) return true; + if (paramTypes.positional && ch === '?' && (nextChar === null || isWhitespace(nextChar))) + return true; if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { if (nextChar !== null && !isNaN(Number(nextChar))) { @@ -513,15 +535,17 @@ function isParameter(ch: Char, state: State, dialect: Dialect, paramTypes?: Para } } - if ((paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) || - (paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh))) { + if ( + (paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) || + (paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) + ) { return true; } - if ((paramTypes.custom && paramTypes.custom.length && isCustomParam(state, paramTypes))) { - return true + if (paramTypes.custom && paramTypes.custom.length && isCustomParam(state, paramTypes)) { + return true; } - + return false; } diff --git a/test/index.spec.ts b/test/index.spec.ts index 200a504..b948b4e 100644 --- a/test/index.spec.ts +++ b/test/index.spec.ts @@ -29,7 +29,7 @@ describe('identify', () => { numbered: ['$'], named: [':'], quoted: [':'], - custom: [ '\\{[a-zA-Z0-9_]+\\}' ] + custom: ['\\{[a-zA-Z0-9_]+\\}'], }; const query = `SELECT * FROM foo WHERE bar = ? AND baz = $1 AND fizz = :fizzz AND buzz = :"buzz buzz" AND foo2 = {fooo}`; @@ -41,10 +41,10 @@ describe('identify', () => { type: 'SELECT', executionType: 'LISTING', parameters: ['?', '$1', ':fizzz', ':"buzz buzz"', '{fooo}'], - tables: [] - } - ]) - }) + tables: [], + }, + ]); + }); it('should identify tables in simple for basic cases', () => { expect( diff --git a/test/tokenizer/index.spec.ts b/test/tokenizer/index.spec.ts index 89e4f4c..beb3a92 100644 --- a/test/tokenizer/index.spec.ts +++ b/test/tokenizer/index.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; import { scanToken } from '../../src/tokenizer'; -import type { Dialect, ParamTypes } from '../../src/defines'; +import type { Dialect, ParamTypes, Token } from '../../src/defines'; describe('scan', () => { const initState = (input: string) => ({ @@ -380,18 +380,19 @@ describe('scan', () => { describe('custom parameters', () => { it('should allow positional parameters for all dialects', () => { const paramTypes: ParamTypes = { - positional: true + positional: true, }; const expected = { type: 'parameter', value: '?', start: 0, - end: 0 + end: 0, }; - - (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + ( + ['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array + ).forEach((dialect) => { [ { actual: scanToken(initState('?'), dialect, paramTypes), @@ -403,7 +404,7 @@ describe('scan', () => { it('should allow numeric parameters for all dialects', () => { const paramTypes: ParamTypes = { - numbered: ["$", "?", ":"] + numbered: ['$', '?', ':'], }; const expected = [ @@ -411,29 +412,31 @@ describe('scan', () => { type: 'parameter', value: '$1', start: 0, - end: 1 + end: 1, }, { type: 'parameter', value: '?1', start: 0, - end: 1 + end: 1, }, { type: 'parameter', value: ':1', start: 0, - end: 1 + end: 1, }, { type: 'parameter', value: 'unknown', start: 0, - end: 8 - } + end: 8, + }, ]; - (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + ( + ['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array + ).forEach((dialect) => { [ { actual: scanToken(initState('$1'), dialect, paramTypes), @@ -449,15 +452,15 @@ describe('scan', () => { }, { actual: scanToken(initState('$123hello'), dialect, paramTypes), // won't recognize - expected: expected[3] - } + expected: expected[3], + }, ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); }); }); it('should allow named parameters for all dialects', () => { const paramTypes: ParamTypes = { - named: ["$", "@", ":"] + named: ['$', '@', ':'], }; const expected = [ @@ -465,29 +468,31 @@ describe('scan', () => { type: 'parameter', value: '$namedParam', start: 0, - end: 10 + end: 10, }, { type: 'parameter', value: '@namedParam', start: 0, - end: 10 + end: 10, }, { type: 'parameter', value: ':namedParam', start: 0, - end: 10 + end: 10, }, { type: 'parameter', value: '$123hello', // allow starting with a number start: 0, - end: 8 - } + end: 8, + }, ]; - (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + ( + ['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array + ).forEach((dialect) => { [ { actual: scanToken(initState('$namedParam'), dialect, paramTypes), @@ -503,78 +508,87 @@ describe('scan', () => { }, { actual: scanToken(initState('$123hello'), dialect, paramTypes), - expected: expected[3] - } + expected: expected[3], + }, ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); - }) + }); }); // this test will need a refactor depending on how we want to implement quotes it('should allow quoted parameters for all dialects', () => { const paramTypes: ParamTypes = { - quoted: ["$", "@", ":"] + quoted: ['$', '@', ':'], }; - const expected = [ + const expected: Array = [ { type: 'parameter', value: '$', start: 0, - end: 14 + end: 14, }, { type: 'parameter', value: '@', start: 0, - end: 14 + end: 14, }, { type: 'parameter', value: ':', start: 0, - end: 14 - } + end: 14, + }, ]; - ([ - { dialect: 'mssql', quotes: ['""', '[]'] }, - { dialect: 'psql', quotes: ['""', '``'] }, - { dialect: 'oracle', quotes: ['""', '``']}, - { dialect: 'bigquery', quotes: ['""', '``']}, - { dialect: 'sqlite', quotes: ['""', '``']}, - { dialect: 'mysql', quotes: ['""', '``']}, - { dialect: 'generic', quotes: ['""', '``']}, - ] as Array<{dialect: Dialect, quotes: Array}>).forEach(({dialect, quotes}) => { - const dialectExpected = expected.map((exp) => { - return quotes.map((quote) => { - return { - ...exp, - value: `${exp.value}${quote[0]}quoted param${quote[1]}` - } - }) - }).flat(); - dialectExpected.map((expected) => ({ - actual: scanToken(initState(expected.value), dialect, paramTypes), - expected - })).forEach(({ actual, expected }) => expect(actual).to.eql(expected)); - }) + ( + [ + { dialect: 'mssql', quotes: ['""', '[]'] }, + { dialect: 'psql', quotes: ['""', '``'] }, + { dialect: 'oracle', quotes: ['""', '``'] }, + { dialect: 'bigquery', quotes: ['""', '``'] }, + { dialect: 'sqlite', quotes: ['""', '``'] }, + { dialect: 'mysql', quotes: ['""', '``'] }, + { dialect: 'generic', quotes: ['""', '``'] }, + ] as Array<{ dialect: Dialect; quotes: Array }> + ).forEach(({ dialect, quotes }) => { + const dialectExpected = Array.prototype.concat.apply( + [], + expected.map((exp) => { + return quotes.map((quote) => { + return { + ...exp, + value: `${exp.value}${quote[0]}quoted param${quote[1]}`, + }; + }); + }), + ); + dialectExpected + .map((expected: Token) => ({ + actual: scanToken(initState(expected.value), dialect, paramTypes), + expected, + })) + .forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + }); }); it('should allow custom parameters for all dialects', () => { const paramTypes: ParamTypes = { - custom: [ '\\{[a-zA-Z0-9_]+\\}' ] + custom: ['\\{[a-zA-Z0-9_]+\\}'], }; const expected = { type: 'parameter', value: '{namedParam}', start: 0, - end: 11 + end: 11, }; - (['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array).forEach((dialect) => { + ( + ['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array + ).forEach((dialect) => { expect(scanToken(initState('{namedParam}'), dialect, paramTypes)).to.eql(expected); - }) + }); }); it('should not have collision between param types', () => { @@ -583,7 +597,7 @@ describe('scan', () => { numbered: [':'], named: [':'], quoted: [':'], - custom: [] + custom: [], }; const expected = [ @@ -591,32 +605,32 @@ describe('scan', () => { type: 'parameter', value: '?', start: 0, - end: 0 + end: 0, }, { type: 'parameter', value: ':123', start: 0, - end: 3 + end: 3, }, { type: 'parameter', value: ':123hello', start: 0, - end: 8 + end: 8, }, { type: 'parameter', value: ':"named param"', start: 0, - end: 13 - } + end: 13, + }, ]; expected.forEach((expected) => { expect(scanToken(initState(expected.value), 'mssql', paramTypes)).to.eql(expected); - }) - }) + }); + }); }); }); }); From 7368b79624702253f676d8a82c9c26572d8e02f4 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Thu, 18 Apr 2024 21:17:29 -0600 Subject: [PATCH 04/16] all tests working again --- src/index.ts | 15 ++- src/parser.ts | 22 ++++ src/tokenizer.ts | 117 +++++---------------- test/parser/single-statements.spec.ts | 10 +- test/tokenizer/index.spec.ts | 143 ++++++++++++++++++-------- 5 files changed, 164 insertions(+), 143 deletions(-) diff --git a/src/index.ts b/src/index.ts index fb34955..9a9f299 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ -import { parse, EXECUTION_TYPES } from './parser'; -import { DIALECTS } from './defines'; +import { parse, EXECUTION_TYPES, defaultParamTypesFor } from './parser'; +import { DIALECTS, ParamTypes } from './defines'; import type { ExecutionType, IdentifyOptions, IdentifyResult, StatementType } from './defines'; export type { @@ -21,7 +21,16 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify throw new Error(`Unknown dialect. Allowed values: ${DIALECTS.join(', ')}`); } - const result = parse(query, isStrict, dialect, options.identifyTables, options.paramTypes); + let paramTypes: ParamTypes; + + // Default parameter types for each dialect + if (options.paramTypes) { + paramTypes = options.paramTypes; + } else { + paramTypes = defaultParamTypesFor(dialect); + } + + const result = parse(query, isStrict, dialect, options.identifyTables, paramTypes); const sort = dialect === 'psql' && !options.paramTypes; return result.body.map((statement) => { diff --git a/src/parser.ts b/src/parser.ts index 57bbff5..9d48a7d 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1015,3 +1015,25 @@ function stateMachineStatementParser( }, }; } + +export function defaultParamTypesFor(dialect: Dialect): ParamTypes { + if (dialect === 'psql') { + return { + numbered: ['$'], + }; + } else if (dialect === 'mssql') { + return { + named: [':'], + }; + } else if (dialect === 'bigquery') { + return { + positional: true, + named: ['@'], + quoted: ['@'], + }; + } else { + return { + positional: true, + }; + } +} diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 94ddfa6..e6b2816 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -79,7 +79,7 @@ const ENDTOKENS: Record = { export function scanToken( state: State, dialect: Dialect = 'generic', - paramTypes?: ParamTypes, + paramTypes: ParamTypes = { positional: true }, ): Token { const ch = read(state); @@ -99,7 +99,7 @@ export function scanToken( return scanString(state, ENDTOKENS[ch]); } - if (isParameter(ch, state, dialect, paramTypes)) { + if (isParameter(ch, state, paramTypes)) { return scanParameter(state, dialect, paramTypes); } @@ -268,19 +268,19 @@ function getCustomParam(state: State, paramTypes: ParamTypes): string | null | u return matches ? matches[0] : null; } -function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): Token { +function scanParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): Token { const curCh: any = state.input[state.start]; const nextChar = peek(state); let matched = false; if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { const endIndex = state.input - .slice(state.start) + .slice(state.start + 1) .split('') - .findIndex((val) => isWhitespace(val)); + .findIndex((val) => /^\W+/.test(val)); const maybeNumbers = state.input.slice( state.start + 1, - endIndex > 0 ? state.start + endIndex : state.end + 1, + endIndex > 0 ? state.start + endIndex + 1 : state.end + 1, ); if (nextChar !== null && !isNaN(Number(nextChar)) && /^\d+$/.test(maybeNumbers)) { let nextChar: Char = null; @@ -312,8 +312,9 @@ function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTy while ( (isAlphaNumeric(peek(state)) || peek(state) === ' ') && peek(state) != ENDTOKENS[quoteChar] - ) + ) { read(state); + } // read the end quote read(state); @@ -331,7 +332,7 @@ function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTy } } - if (!matched && !paramTypes.positional) { + if (!matched && !paramTypes.positional && curCh !== '?') { // not positional, panic return { type: 'parameter', @@ -350,60 +351,6 @@ function scanCustomParameter(state: State, dialect: Dialect, paramTypes: ParamTy }; } -function scanParameter(state: State, dialect: Dialect, paramTypes?: ParamTypes): Token { - // user has defined wanted param types, so we only evaluate them - if (paramTypes) { - return scanCustomParameter(state, dialect, paramTypes); - } - - if (['mysql', 'generic', 'sqlite'].includes(dialect)) { - return { - type: 'parameter', - value: state.input.slice(state.start, state.position + 1), - start: state.start, - end: state.start, - }; - } - - if (dialect === 'psql') { - let nextChar: Char; - - do { - nextChar = read(state); - } while (nextChar !== null && !isNaN(Number(nextChar)) && !isWhitespace(nextChar)); - - if (nextChar !== null) unread(state); - - const value = state.input.slice(state.start, state.position + 1); - - return { - type: 'parameter', - value, - start: state.start, - end: state.start + value.length - 1, - }; - } - - if (dialect === 'mssql') { - while (isAlphaNumeric(peek(state))) read(state); - - const value = state.input.slice(state.start, state.position + 1); - return { - type: 'parameter', - value, - start: state.start, - end: state.start + value.length - 1, - }; - } - - return { - type: 'parameter', - value: 'unknown', - start: state.start, - end: state.end, - }; -} - function scanCommentBlock(state: State): Token { let nextChar: Char = ''; let prevChar: Char; @@ -522,44 +469,30 @@ function isCustomParam(state: State, paramTypes: ParamTypes): boolean | undefine }); } -function isParameter(ch: Char, state: State, dialect: Dialect, paramTypes?: ParamTypes): boolean { - if (paramTypes && ch !== null) { - const curCh: any = ch; - const nextChar = peek(state); - if (paramTypes.positional && ch === '?' && (nextChar === null || isWhitespace(nextChar))) - return true; - - if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { - if (nextChar !== null && !isNaN(Number(nextChar))) { - return true; - } - } - - if ( - (paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) || - (paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) - ) { - return true; - } +function isParameter(ch: Char, state: State, paramTypes: ParamTypes): boolean { + const curCh: any = ch; + const nextChar = peek(state); + if (paramTypes.positional && ch === '?') + return true; - if (paramTypes.custom && paramTypes.custom.length && isCustomParam(state, paramTypes)) { + if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { + if (nextChar !== null && !isNaN(Number(nextChar))) { return true; } + } - return false; + if ( + (paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) || + (paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) + ) { + return true; } - let pStart = '?'; // ansi standard - sqlite, mysql - if (dialect === 'psql') { - pStart = '$'; - const nextChar = peek(state); - if (nextChar === null || isNaN(Number(nextChar))) { - return false; - } + if (paramTypes.custom && paramTypes.custom.length && isCustomParam(state, paramTypes)) { + return true; } - if (dialect === 'mssql') pStart = ':'; - return ch === pStart; + return false; } function isDollarQuotedString(state: State): boolean { diff --git a/test/parser/single-statements.spec.ts b/test/parser/single-statements.spec.ts index b0a15cc..32c9c36 100644 --- a/test/parser/single-statements.spec.ts +++ b/test/parser/single-statements.spec.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { aggregateUnknownTokens } from '../spec-helper'; -import { parse } from '../../src/parser'; +import { defaultParamTypesFor, parse } from '../../src/parser'; import { Token } from '../../src/defines'; describe('parser', () => { @@ -725,7 +725,7 @@ describe('parser', () => { }); it('should extract PSQL parameters', () => { - const actual = parse('select x from a where x = $1', true, 'psql'); + const actual = parse('select x from a where x = $1', true, 'psql', false, defaultParamTypesFor('psql')); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { @@ -752,7 +752,7 @@ describe('parser', () => { }); it('should extract multiple PSQL parameters', () => { - const actual = parse('select x from a where x = $1 and y = $2', true, 'psql'); + const actual = parse('select x from a where x = $1 and y = $2', true, 'psql', false, defaultParamTypesFor('psql')); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { @@ -791,7 +791,7 @@ describe('parser', () => { }); it('should extract mssql parameters', () => { - const actual = parse('select x from a where x = :foo', true, 'mssql'); + const actual = parse('select x from a where x = :foo', true, 'mssql', false, defaultParamTypesFor('mssql')); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { @@ -856,7 +856,7 @@ describe('parser', () => { }); it('should extract multiple mssql parameters', () => { - const actual = parse('select x from a where x = :foo and y = :bar', true, 'mssql'); + const actual = parse('select x from a where x = :foo and y = :bar', true, 'mssql', false, defaultParamTypesFor('mssql')); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { diff --git a/test/tokenizer/index.spec.ts b/test/tokenizer/index.spec.ts index beb3a92..b5f5d6e 100644 --- a/test/tokenizer/index.spec.ts +++ b/test/tokenizer/index.spec.ts @@ -1,6 +1,7 @@ import { expect } from 'chai'; import { scanToken } from '../../src/tokenizer'; import type { Dialect, ParamTypes, Token } from '../../src/defines'; +import { defaultParamTypesFor } from '../../src/parser'; describe('scan', () => { const initState = (input: string) => ({ @@ -274,7 +275,11 @@ describe('scan', () => { ].forEach(([ch, dialect]) => { it(`scans just ${ch} as parameter for ${dialect}`, () => { const input = `${ch}`; - const actual = scanToken(initState(input), dialect as Dialect); + const actual = scanToken( + initState(input), + dialect as Dialect, + defaultParamTypesFor(dialect as Dialect), + ); const expected = { type: 'parameter', value: input, @@ -286,7 +291,7 @@ describe('scan', () => { }); it('does not scan just $ as parameter for psql', () => { const input = '$'; - const actual = scanToken(initState(input), 'psql'); + const actual = scanToken(initState(input), 'psql', defaultParamTypesFor('psql')); const expected = { type: 'unknown', value: input, @@ -304,7 +309,11 @@ describe('scan', () => { ].forEach(([ch, dialect]) => { it(`should only scan ${ch} from ${ch}1 for ${dialect}`, () => { const input = `${ch}1`; - const actual = scanToken(initState(input), dialect as Dialect); + const actual = scanToken( + initState(input), + dialect as Dialect, + defaultParamTypesFor(dialect as Dialect), + ); const expected = { type: 'parameter', value: ch, @@ -320,7 +329,11 @@ describe('scan', () => { ].forEach(([ch, dialect]) => { it(`should scan ${ch}1 for ${dialect}`, () => { const input = `${ch}1`; - const actual = scanToken(initState(input), dialect as Dialect); + const actual = scanToken( + initState(input), + dialect as Dialect, + defaultParamTypesFor(dialect as Dialect), + ); const expected = { type: 'parameter', value: input, @@ -333,7 +346,7 @@ describe('scan', () => { it('should not scan $a for psql', () => { const input = '$a'; - const actual = scanToken(initState(input), 'psql'); + const actual = scanToken(initState(input), 'psql', defaultParamTypesFor('psql')); const expected = { type: 'unknown', value: '$', @@ -344,7 +357,7 @@ describe('scan', () => { }); it('should not include trailing non-numbers for psql', () => { - const actual = scanToken(initState('$1,'), 'psql'); + const actual = scanToken(initState('$1,'), 'psql', defaultParamTypesFor('psql')); const expected = { type: 'parameter', value: '$1', @@ -355,9 +368,10 @@ describe('scan', () => { }); it('should not include trailing non-alphanumerics for mssql', () => { + const paramTypes = defaultParamTypesFor('mssql'); [ { - actual: scanToken(initState(':one,'), 'mssql'), + actual: scanToken(initState(':one,'), 'mssql', paramTypes), expected: { type: 'parameter', value: ':one', @@ -366,7 +380,7 @@ describe('scan', () => { }, }, { - actual: scanToken(initState(':two)'), 'mssql'), + actual: scanToken(initState(':two)'), 'mssql', paramTypes), expected: { type: 'parameter', value: ':two', @@ -378,17 +392,19 @@ describe('scan', () => { }); describe('custom parameters', () => { - it('should allow positional parameters for all dialects', () => { - const paramTypes: ParamTypes = { + describe('positional parameters', () => { + const paramTypes = { positional: true, }; - const expected = { - type: 'parameter', - value: '?', - start: 0, - end: 0, - }; + const expected = [ + { + type: 'parameter', + value: '?', + start: 0, + end: 0, + }, + ]; ( ['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array @@ -396,13 +412,17 @@ describe('scan', () => { [ { actual: scanToken(initState('?'), dialect, paramTypes), - expected, + expected: expected[0], }, - ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + ].forEach(({ actual, expected }) => { + it(`should allow positional parameters for ${dialect}`, () => { + expect(actual).to.eql(expected); + }); + }); }); }); - it('should allow numeric parameters for all dialects', () => { + describe('numeric parameters', () => { const paramTypes: ParamTypes = { numbered: ['$', '?', ':'], }; @@ -441,24 +461,32 @@ describe('scan', () => { { actual: scanToken(initState('$1'), dialect, paramTypes), expected: expected[0], + description: '$ numeric', }, { actual: scanToken(initState('?1'), dialect, paramTypes), expected: expected[1], + description: '? numeric', }, { actual: scanToken(initState(':1'), dialect, paramTypes), expected: expected[2], + description: ': numeric', }, { actual: scanToken(initState('$123hello'), dialect, paramTypes), // won't recognize expected: expected[3], + description: 'numeric trailing alpha', }, - ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + ].forEach(({ actual, expected, description }) => { + it(`should allow numeric parameters for ${dialect} - ${description}`, () => { + expect(actual).to.eql(expected); + }); + }); }); }); - it('should allow named parameters for all dialects', () => { + describe('named parameters', () => { const paramTypes: ParamTypes = { named: ['$', '@', ':'], }; @@ -497,30 +525,37 @@ describe('scan', () => { { actual: scanToken(initState('$namedParam'), dialect, paramTypes), expected: expected[0], + description: '$ named', }, { actual: scanToken(initState('@namedParam'), dialect, paramTypes), expected: expected[1], + description: '@ named', }, { actual: scanToken(initState(':namedParam'), dialect, paramTypes), expected: expected[2], + description: ': named', }, { actual: scanToken(initState('$123hello'), dialect, paramTypes), expected: expected[3], + description: 'named starting with numbers', }, - ].forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + ].forEach(({ actual, expected, description }) => { + it(`should allow named parameters for ${dialect} - ${description}`, () => { + expect(actual).to.eql(expected); + }); + }); }); }); - // this test will need a refactor depending on how we want to implement quotes - it('should allow quoted parameters for all dialects', () => { + describe('quoted parameters', () => { const paramTypes: ParamTypes = { quoted: ['$', '@', ':'], }; - const expected: Array = [ + const expected = [ { type: 'parameter', value: '$', @@ -557,49 +592,63 @@ describe('scan', () => { expected.map((exp) => { return quotes.map((quote) => { return { - ...exp, - value: `${exp.value}${quote[0]}quoted param${quote[1]}`, + expected: { + ...exp, + value: `${exp.value}${quote[0]}quoted param${quote[1]}`, + }, + description: `${exp.value} quoted with ${quote[0]}`, }; }); }), ); dialectExpected - .map((expected: Token) => ({ - actual: scanToken(initState(expected.value), dialect, paramTypes), - expected, + .map(({ expected, description }) => ({ + actual: scanToken(initState((expected as Token).value), dialect, paramTypes), + expected: expected as Token, + description: description as string, })) - .forEach(({ actual, expected }) => expect(actual).to.eql(expected)); + .forEach(({ actual, expected, description }) => { + it(`should allow quoted parameters for ${dialect} - ${description}`, () => { + expect(actual).to.eql(expected); + }); + }); }); }); - it('should allow custom parameters for all dialects', () => { + describe('custom parameters', () => { const paramTypes: ParamTypes = { custom: ['\\{[a-zA-Z0-9_]+\\}'], }; - const expected = { - type: 'parameter', - value: '{namedParam}', - start: 0, - end: 11, - }; + const expected = [ + { + type: 'parameter', + value: '{namedParam}', + start: 0, + end: 11, + }, + ]; ( ['mssql', 'psql', 'oracle', 'bigquery', 'sqlite', 'mysql', 'generic'] as Array ).forEach((dialect) => { - expect(scanToken(initState('{namedParam}'), dialect, paramTypes)).to.eql(expected); + it(`should allow custom parameters for ${dialect}`, () => { + expect(scanToken(initState('{namedParam}'), dialect, paramTypes)).to.eql(expected[0]); + }); }); }); - it('should not have collision between param types', () => { + describe('should not have collision between param types', () => { const paramTypes: ParamTypes = { positional: true, numbered: [':'], named: [':'], quoted: [':'], - custom: [], + custom: ['\\{[a-zA-Z0-9_]+\\}'], }; + const type = ['positional', 'numeric', 'named', 'quoted', 'custom']; + const expected = [ { type: 'parameter', @@ -625,10 +674,18 @@ describe('scan', () => { start: 0, end: 13, }, + { + type: 'parameter', + value: '{namedParam}', + start: 0, + end: 11, + }, ]; - expected.forEach((expected) => { - expect(scanToken(initState(expected.value), 'mssql', paramTypes)).to.eql(expected); + expected.forEach((expected, index) => { + it(`parameter types don't collide, finds ${type[index]}`, () => { + expect(scanToken(initState(expected.value), 'mssql', paramTypes)).to.eql(expected); + }); }); }); }); From a3f8b51f92f14e32bd7ab3270a687e8e3ab1128f Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Thu, 18 Apr 2024 21:20:17 -0600 Subject: [PATCH 05/16] some linting complaints, oops --- src/tokenizer.ts | 3 +-- test/parser/single-statements.spec.ts | 32 +++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index e6b2816..e4da3af 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -472,8 +472,7 @@ function isCustomParam(state: State, paramTypes: ParamTypes): boolean | undefine function isParameter(ch: Char, state: State, paramTypes: ParamTypes): boolean { const curCh: any = ch; const nextChar = peek(state); - if (paramTypes.positional && ch === '?') - return true; + if (paramTypes.positional && ch === '?') return true; if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { if (nextChar !== null && !isNaN(Number(nextChar))) { diff --git a/test/parser/single-statements.spec.ts b/test/parser/single-statements.spec.ts index 32c9c36..4fa5b0b 100644 --- a/test/parser/single-statements.spec.ts +++ b/test/parser/single-statements.spec.ts @@ -725,7 +725,13 @@ describe('parser', () => { }); it('should extract PSQL parameters', () => { - const actual = parse('select x from a where x = $1', true, 'psql', false, defaultParamTypesFor('psql')); + const actual = parse( + 'select x from a where x = $1', + true, + 'psql', + false, + defaultParamTypesFor('psql'), + ); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { @@ -752,7 +758,13 @@ describe('parser', () => { }); it('should extract multiple PSQL parameters', () => { - const actual = parse('select x from a where x = $1 and y = $2', true, 'psql', false, defaultParamTypesFor('psql')); + const actual = parse( + 'select x from a where x = $1 and y = $2', + true, + 'psql', + false, + defaultParamTypesFor('psql'), + ); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { @@ -791,7 +803,13 @@ describe('parser', () => { }); it('should extract mssql parameters', () => { - const actual = parse('select x from a where x = :foo', true, 'mssql', false, defaultParamTypesFor('mssql')); + const actual = parse( + 'select x from a where x = :foo', + true, + 'mssql', + false, + defaultParamTypesFor('mssql'), + ); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { @@ -856,7 +874,13 @@ describe('parser', () => { }); it('should extract multiple mssql parameters', () => { - const actual = parse('select x from a where x = :foo and y = :bar', true, 'mssql', false, defaultParamTypesFor('mssql')); + const actual = parse( + 'select x from a where x = :foo and y = :bar', + true, + 'mssql', + false, + defaultParamTypesFor('mssql'), + ); actual.tokens = aggregateUnknownTokens(actual.tokens); const expected: Token[] = [ { From 47c2407576637babb6ce3a4b25cfddd2c36cf045 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Fri, 19 Apr 2024 10:30:59 -0600 Subject: [PATCH 06/16] add test for overriding defaults --- test/index.spec.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/index.spec.ts b/test/index.spec.ts index b948b4e..ba9f2f9 100644 --- a/test/index.spec.ts +++ b/test/index.spec.ts @@ -46,6 +46,26 @@ describe('identify', () => { ]); }); + it('custom params should override defaults for dialect', () => { + const paramTypes: ParamTypes = { + positional: true + }; + + const query = 'SELECT * FROM foo WHERE bar = $1 AND bar = :named AND fizz = :`quoted`'; + + expect(identify(query, { dialect: 'psql', paramTypes })).to.eql([ + { + start: 0, + end: 69, + text: query, + type: 'SELECT', + executionType: 'LISTING', + parameters: [], + tables: [] + } + ]) + }) + it('should identify tables in simple for basic cases', () => { expect( identify('SELECT * FROM foo JOIN bar ON foo.id = bar.id', { identifyTables: true }), From a30fb2afaa6222a64058b11fe41c370b9b0838ef Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Fri, 19 Apr 2024 10:31:23 -0600 Subject: [PATCH 07/16] lints ugh --- test/index.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/index.spec.ts b/test/index.spec.ts index ba9f2f9..d2ef657 100644 --- a/test/index.spec.ts +++ b/test/index.spec.ts @@ -48,7 +48,7 @@ describe('identify', () => { it('custom params should override defaults for dialect', () => { const paramTypes: ParamTypes = { - positional: true + positional: true, }; const query = 'SELECT * FROM foo WHERE bar = $1 AND bar = :named AND fizz = :`quoted`'; @@ -61,10 +61,10 @@ describe('identify', () => { type: 'SELECT', executionType: 'LISTING', parameters: [], - tables: [] - } - ]) - }) + tables: [], + }, + ]); + }); it('should identify tables in simple for basic cases', () => { expect( From 6d42571ed3bec6e516e96009e2db61ce8e8dbdb9 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Wed, 24 Apr 2024 09:52:21 -0600 Subject: [PATCH 08/16] return unknown type if not param --- src/tokenizer.ts | 8 ++++---- test/tokenizer/index.spec.ts | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index e4da3af..a7035a5 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -331,18 +331,18 @@ function scanParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): matched = true; } } + const value = state.input.slice(state.start, state.position + 1); if (!matched && !paramTypes.positional && curCh !== '?') { // not positional, panic return { - type: 'parameter', - value: 'unknown', + type: 'unknown', + value: value, start: state.start, - end: state.end, + end: state.start + value.length - 1, }; } - const value = state.input.slice(state.start, state.position + 1); return { type: 'parameter', value, diff --git a/test/tokenizer/index.spec.ts b/test/tokenizer/index.spec.ts index b5f5d6e..cafa4eb 100644 --- a/test/tokenizer/index.spec.ts +++ b/test/tokenizer/index.spec.ts @@ -447,10 +447,10 @@ describe('scan', () => { end: 1, }, { - type: 'parameter', - value: 'unknown', + type: 'unknown', + value: '$', start: 0, - end: 8, + end: 0, }, ]; From 4facca26039c3894489bd9c4d939599e09064874 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Wed, 24 Apr 2024 10:15:17 -0600 Subject: [PATCH 09/16] adhere to spec for default params --- src/parser.ts | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 9d48a7d..06150d7 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1017,23 +1017,30 @@ function stateMachineStatementParser( } export function defaultParamTypesFor(dialect: Dialect): ParamTypes { - if (dialect === 'psql') { - return { - numbered: ['$'], - }; - } else if (dialect === 'mssql') { - return { - named: [':'], - }; - } else if (dialect === 'bigquery') { - return { - positional: true, - named: ['@'], - quoted: ['@'], - }; - } else { - return { - positional: true, - }; + switch (dialect) { + case 'psql': + return { + numbered: ['$'], + }; + case 'mssql': + return { + named: [':'], + }; + case 'bigquery': + return { + positional: true, + named: ['@'], + quoted: ['@'], + }; + case 'sqlite': + return { + positional: true, + numbered: ['?'], + named: [':', '@'], + }; + default: + return { + positional: true, + }; } } From e4f1677824d11d531f20387c154854ae3ef14028 Mon Sep 17 00:00:00 2001 From: Day Matchullis Date: Wed, 24 Apr 2024 10:18:20 -0600 Subject: [PATCH 10/16] remove obsolete test --- test/tokenizer/index.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/tokenizer/index.spec.ts b/test/tokenizer/index.spec.ts index cafa4eb..7195735 100644 --- a/test/tokenizer/index.spec.ts +++ b/test/tokenizer/index.spec.ts @@ -305,7 +305,6 @@ describe('scan', () => { [ ['?', 'generic'], ['?', 'mysql'], - ['?', 'sqlite'], ].forEach(([ch, dialect]) => { it(`should only scan ${ch} from ${ch}1 for ${dialect}`, () => { const input = `${ch}1`; From 99870c408095cd5b99ed730eb8be385f5f72683f Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Tue, 30 Apr 2024 16:04:22 -0400 Subject: [PATCH 11/16] remove usage of any Signed-off-by: Matthew Peveler --- src/tokenizer.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index a7035a5..58c262c 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -269,11 +269,11 @@ function getCustomParam(state: State, paramTypes: ParamTypes): string | null | u } function scanParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): Token { - const curCh: any = state.input[state.start]; + const curCh = state.input[state.start]; const nextChar = peek(state); let matched = false; - if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { + if (paramTypes.numbered?.length && paramTypes.numbered.some((type) => type === curCh)) { const endIndex = state.input .slice(state.start + 1) .split('') @@ -293,19 +293,14 @@ function scanParameter(state: State, dialect: Dialect, paramTypes: ParamTypes): } } - if (!matched && paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) { + if (!matched && paramTypes.named?.length && paramTypes.named.some((type) => type === curCh)) { if (!isQuotedIdentifier(nextChar, dialect)) { while (isAlphaNumeric(peek(state))) read(state); matched = true; } } - if ( - !matched && - paramTypes.quoted && - paramTypes.quoted.length && - paramTypes.quoted.includes(curCh) - ) { + if (!matched && paramTypes.quoted?.length && paramTypes.quoted.some((type) => type === curCh)) { if (isQuotedIdentifier(nextChar, dialect)) { const quoteChar = read(state) as string; // end when we reach the end quote @@ -462,32 +457,37 @@ function isString(ch: Char, dialect: Dialect): boolean { return stringStart.includes(ch); } -function isCustomParam(state: State, paramTypes: ParamTypes): boolean | undefined { - return paramTypes?.custom?.some((regex) => { +function isCustomParam( + state: State, + customParamType: NonNullable, +): boolean | undefined { + return customParamType.some((regex) => { const reg = new RegExp(`^(?:${regex})`, 'uy'); return reg.test(state.input.slice(state.start)); }); } function isParameter(ch: Char, state: State, paramTypes: ParamTypes): boolean { - const curCh: any = ch; + if (!ch) { + return false; + } const nextChar = peek(state); if (paramTypes.positional && ch === '?') return true; - if (paramTypes.numbered && paramTypes.numbered.length && paramTypes.numbered.includes(curCh)) { + if (paramTypes.numbered?.length && paramTypes.numbered.some((type) => ch === type)) { if (nextChar !== null && !isNaN(Number(nextChar))) { return true; } } if ( - (paramTypes.named && paramTypes.named.length && paramTypes.named.includes(curCh)) || - (paramTypes.quoted && paramTypes.quoted.length && paramTypes.quoted.includes(curCh)) + (paramTypes.named?.length && paramTypes.named.some((type) => type === ch)) || + (paramTypes.quoted?.length && paramTypes.quoted.some((type) => type === ch)) ) { return true; } - if (paramTypes.custom && paramTypes.custom.length && isCustomParam(state, paramTypes)) { + if (paramTypes.custom?.length && isCustomParam(state, paramTypes.custom)) { return true; } From 891b3e7f486987948db496efe0a13b0713a786b3 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Tue, 30 Apr 2024 16:05:06 -0400 Subject: [PATCH 12/16] Use [] over Array for TS --- src/defines.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/defines.ts b/src/defines.ts index 012331b..02eec84 100644 --- a/src/defines.ts +++ b/src/defines.ts @@ -78,11 +78,11 @@ export type ExecutionType = 'LISTING' | 'MODIFICATION' | 'INFORMATION' | 'ANON_B export interface ParamTypes { positional?: boolean; - numbered?: Array<'?' | ':' | '$'>; - named?: Array<':' | '@' | '$'>; - quoted?: Array<':' | '@' | '$'>; + numbered?: ('?' | ':' | '$')[]; + named?: (':' | '@' | '$')[]; + quoted?: (':' | '@' | '$')[]; // regex for identifying that it is a param - custom?: Array; + custom?: string[]; } export interface IdentifyOptions { From 398661718f229770a0d7f8436e9d9d4ad340aea8 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Tue, 30 Apr 2024 16:05:31 -0400 Subject: [PATCH 13/16] Update src/index.ts --- src/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9a9f299..ec616d6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,11 +24,7 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify let paramTypes: ParamTypes; // Default parameter types for each dialect - if (options.paramTypes) { - paramTypes = options.paramTypes; - } else { - paramTypes = defaultParamTypesFor(dialect); - } + paramTypes = options.paramTypes || defaultParamTypesFor(dialect); const result = parse(query, isStrict, dialect, options.identifyTables, paramTypes); const sort = dialect === 'psql' && !options.paramTypes; From 6e7eb1eaa004acb441516b7bb6378bd5f11cbde5 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Tue, 30 Apr 2024 16:08:54 -0400 Subject: [PATCH 14/16] Update tokenizer.ts --- src/tokenizer.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 58c262c..1214b72 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -457,10 +457,7 @@ function isString(ch: Char, dialect: Dialect): boolean { return stringStart.includes(ch); } -function isCustomParam( - state: State, - customParamType: NonNullable, -): boolean | undefined { +function isCustomParam(state: State, customParamType: NonNullable): boolean { return customParamType.some((regex) => { const reg = new RegExp(`^(?:${regex})`, 'uy'); return reg.test(state.input.slice(state.start)); From f0fcfb1864f24fd8df1fde83eee67efbef22d693 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Tue, 30 Apr 2024 16:10:12 -0400 Subject: [PATCH 15/16] Update index.ts --- src/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index ec616d6..b21f9c5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,10 +21,8 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify throw new Error(`Unknown dialect. Allowed values: ${DIALECTS.join(', ')}`); } - let paramTypes: ParamTypes; - // Default parameter types for each dialect - paramTypes = options.paramTypes || defaultParamTypesFor(dialect); + const paramTypes = options.paramTypes || defaultParamTypesFor(dialect); const result = parse(query, isStrict, dialect, options.identifyTables, paramTypes); const sort = dialect === 'psql' && !options.paramTypes; From ee17578e7ffa61115935ef6f353fc7ca48b08451 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Tue, 30 Apr 2024 16:11:19 -0400 Subject: [PATCH 16/16] Update index.ts --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index b21f9c5..f600339 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import { parse, EXECUTION_TYPES, defaultParamTypesFor } from './parser'; -import { DIALECTS, ParamTypes } from './defines'; +import { DIALECTS } from './defines'; import type { ExecutionType, IdentifyOptions, IdentifyResult, StatementType } from './defines'; export type {