diff --git a/packages/build/package.json b/packages/build/package.json index 79e6f386..2719b8c5 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/build", "description": "VulcanSQL package for building projects", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -22,6 +22,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/catalog-server/package.json b/packages/catalog-server/package.json index a721112b..af8bb3aa 100644 --- a/packages/catalog-server/package.json +++ b/packages/catalog-server/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/catalog-server", "description": "Catalog server for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "publishConfig": { "access": "public" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index 0a44fb78..163088d0 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/cli", "description": "CLI tools for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "bin": { "vulcan": "./src/index.js" diff --git a/packages/core/package.json b/packages/core/package.json index f37bff72..213836cb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/core", "description": "Core package of VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" diff --git a/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts index 8c9b3d76..8bf5511b 100644 --- a/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts @@ -1,4 +1,11 @@ import { UniqueFilterBuilder } from './uniqueFilterBuilder'; import { UniqueFilterRunner } from './uniqueFilterRunner'; +import { TypeofBuilder } from './typeofBuilder'; +import { TypeofRunner } from './typeofRunner'; -export default [UniqueFilterBuilder, UniqueFilterRunner]; +export default [ + UniqueFilterBuilder, + UniqueFilterRunner, + TypeofBuilder, + TypeofRunner, +]; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/typeofBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/typeofBuilder.ts new file mode 100644 index 00000000..84698ec0 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/typeofBuilder.ts @@ -0,0 +1,9 @@ +import { + FilterBuilder, + VulcanInternalExtension, +} from '@vulcan-sql/core/models'; + +@VulcanInternalExtension() +export class TypeofBuilder extends FilterBuilder { + public filterName = 'typeof'; +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/typeofRunner.ts b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/typeofRunner.ts new file mode 100644 index 00000000..e38502ff --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/typeofRunner.ts @@ -0,0 +1,15 @@ +import { + FilterRunner, + FilterRunnerTransformOptions, + VulcanInternalExtension, +} from '@vulcan-sql/core/models'; + +@VulcanInternalExtension() +export class TypeofRunner extends FilterRunner { + public filterName = 'typeof'; + public async transform({ + value, + }: FilterRunnerTransformOptions): Promise { + return typeof value; + } +} diff --git a/packages/extension-api-caller/package.json b/packages/extension-api-caller/package.json index 6f2cefe3..c65844ea 100644 --- a/packages/extension-api-caller/package.json +++ b/packages/extension-api-caller/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-api-caller", "description": "Calling APIs to get data from other sources", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "^0.10.1" + "@vulcan-sql/core": "^0.10.2" } } diff --git a/packages/extension-authenticator-canner/package.json b/packages/extension-authenticator-canner/package.json index ddeae8e3..8c88b435 100644 --- a/packages/extension-authenticator-canner/package.json +++ b/packages/extension-authenticator-canner/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-authenticator-canner", "description": "Canner Enterprise authenticator for Vulcan SQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -24,7 +24,7 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0", - "@vulcan-sql/serve": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0", + "@vulcan-sql/serve": "~0.10.2-0" } } diff --git a/packages/extension-dbt/package.json b/packages/extension-dbt/package.json index 5126ca13..dd54294d 100644 --- a/packages/extension-dbt/package.json +++ b/packages/extension-dbt/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-dbt", "description": "Using dbt models form VulcanSQL projects", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-debug-tools/package.json b/packages/extension-debug-tools/package.json index f90c36c3..d637513c 100644 --- a/packages/extension-debug-tools/package.json +++ b/packages/extension-debug-tools/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-debug-tools", "description": "A collection of Vulcan extension debug tools", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -22,9 +22,9 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" }, "devDependencies": { - "@vulcan-sql/test-utility": "~0.10.1-0" + "@vulcan-sql/test-utility": "~0.10.2-0" } } diff --git a/packages/extension-driver-bq/package.json b/packages/extension-driver-bq/package.json index 40898616..2aed23d6 100644 --- a/packages/extension-driver-bq/package.json +++ b/packages/extension-driver-bq/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-bq", "description": "BigQuery driver for Vulcan SQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -24,6 +24,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-canner/package.json b/packages/extension-driver-canner/package.json index 9282a10a..b500f254 100644 --- a/packages/extension-driver-canner/package.json +++ b/packages/extension-driver-canner/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-canner", "description": "Canner Enterprise driver for Vulcan SQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -24,6 +24,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-clickhouse/package.json b/packages/extension-driver-clickhouse/package.json index 09072853..87322cac 100644 --- a/packages/extension-driver-clickhouse/package.json +++ b/packages/extension-driver-clickhouse/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-clickhouse", "description": "Clickhouse driver for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -27,6 +27,6 @@ "@clickhouse/client": "^0.1.1" }, "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-duckdb/package.json b/packages/extension-driver-duckdb/package.json index b488e187..b4b8d32c 100644 --- a/packages/extension-driver-duckdb/package.json +++ b/packages/extension-driver-duckdb/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-duckdb", "description": "duckdb driver for Vulcan SQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts b/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts index e64e3991..28f2c82f 100644 --- a/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts +++ b/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts @@ -104,15 +104,13 @@ export class DuckDBDataSource extends DataSource { // create new connection for each query const parameters = Array.from(bindParams.values()); this.logRequest(firstDataSQL, parameters, options); - const connection = db.connect(); - await this.loadExtensions(connection, configurationParameters); - await this.setExecConfig(connection); if (restDataSQL) this.logRequest(restDataSQL, parameters, options); const [firstData, restDataStream] = await this.acquireData( firstDataSQL, restDataSQL, parameters, - db + db, + configurationParameters ); const readable = this.createReadableStream(firstData, restDataStream); return { @@ -168,15 +166,24 @@ export class DuckDBDataSource extends DataSource { firstDataSql: string, restDataSql: string | undefined, parameters: any[], - db: duckdb.Database + db: duckdb.Database, + configurationParameters: ConfigurationParameters ) { // conn.all() is faster then stream.checkChunk(). // For the small size data we use conn.all() to get the data at once // To limit memory use and prevent server crashes, we will use conn.all() to acquire the initial chunk of data, then conn.stream() to receive the remainder of the data. + const c1 = db.connect(); + const c2 = db.connect(); + await Promise.all([ + await this.loadExtensions(c1, configurationParameters), + await this.setExecConfig(c1), + await this.loadExtensions(c2, configurationParameters), + await this.setExecConfig(c2), + ]); + return await Promise.all([ new Promise((resolve, reject) => { - const c = db.connect(); - c.all( + c1.all( firstDataSql, ...parameters, (err: duckdb.DuckDbError | null, res: duckdb.TableData) => { @@ -190,8 +197,7 @@ export class DuckDBDataSource extends DataSource { new Promise((resolve, reject) => { if (!restDataSql) resolve(undefined); try { - const c = db.connect(); - const result = c.stream(restDataSql, ...parameters); + const result = c2.stream(restDataSql, ...parameters); resolve(result); } catch (err: any) { reject(err); diff --git a/packages/extension-driver-duckdb/src/lib/duckdbExtensionLoader.ts b/packages/extension-driver-duckdb/src/lib/duckdbExtensionLoader.ts index edfaf12e..87dd0cba 100644 --- a/packages/extension-driver-duckdb/src/lib/duckdbExtensionLoader.ts +++ b/packages/extension-driver-duckdb/src/lib/duckdbExtensionLoader.ts @@ -37,36 +37,51 @@ export class DuckDBExtensionLoader { return; } try { - conn.run(`LOAD ${extensionName}`); + await new Promise((resolve, reject) => { + conn.run(`LOAD ${extensionName}`, (err: any) => { + if (err) reject(err); + this.logger.debug('Extension loaded'); + resolve(); + }); + }); } catch (error) { this.logger.debug(`Error when loading extension:${extensionName}`); throw error; } + await Promise.all( + Object.entries(extensionConfigurations).map( + async ([dbParameterName, configurationKey]) => { + const configurationValue = + this.configurations[ + configurationKey as keyof ConfigurationParameters + ]; + // if configuration is not undefined + if (configurationValue !== undefined) { + return await new Promise((resolve, reject) => { + conn.run( + `SET ${dbParameterName}='${configurationValue}'`, + (err: any) => { + if (err) { + this.logger.debug( + `Configuration error "${dbParameterName}": ${err}` + ); + reject(err); + } - Object.entries(extensionConfigurations).forEach( - ([dbParameterName, configurationKey]) => { - const configurationValue = - this.configurations[ - configurationKey as keyof ConfigurationParameters - ]; - // if configuration is not undefined - if (configurationValue !== undefined) { - conn.run( - `SET ${dbParameterName}='${configurationValue}'`, - (err: any) => { - if (err) throw err; - this.logger.debug( - `Configuration error "${dbParameterName}": ${err}` + this.logger.debug( + `Configuration parameter "${dbParameterName}" set` + ); + resolve(); + } ); - } - ); - this.logger.debug(`Configuration parameter "${dbParameterName}" set`); - } else { - this.logger.debug( - `Configuration "${dbParameterName}" has not been set` - ); + }); + } else { + this.logger.debug( + `Configuration "${dbParameterName}" has not been set` + ); + } } - } + ) ); } } diff --git a/packages/extension-driver-duckdb/tests/duckdbDataSource.spec.ts b/packages/extension-driver-duckdb/tests/duckdbDataSource.spec.ts index 4373c060..6a7cb2ab 100644 --- a/packages/extension-driver-duckdb/tests/duckdbDataSource.spec.ts +++ b/packages/extension-driver-duckdb/tests/duckdbDataSource.spec.ts @@ -316,7 +316,9 @@ it('Should print queries without binding when log-queries = true', async () => { profileName: 'mocked-profile', }); // Assert - expect(/select \$1::INTEGER as test/.test(logs.slice(-1)[0][0])).toBe(true); + expect( + logs.flat(2).find((log) => /select \$1::INTEGER as test/.test(log)) + ).not.toBe(undefined); }); it('Should print queries with binding when log-queries = true and log-parameters = true', async () => { @@ -357,8 +359,11 @@ it('Should print queries with binding when log-queries = true and log-parameters profileName: 'mocked-profile', }); // Assert - expect(/select \$1::INTEGER as test/.test(logs.slice(-1)[0][0])).toBe(true); - expect(logs.slice(-1)[0][1]).toEqual([1234]); + expect( + logs.flat(2).find((log) => /select \$1::INTEGER as test/.test(log)) + ).not.toBe(undefined); + + expect(logs.flat(2)).toContain(1234); }); it('Should share db instances for same path besides in-memory only db', async () => { diff --git a/packages/extension-driver-ksqldb/package.json b/packages/extension-driver-ksqldb/package.json index 7efef4df..ebf52fc9 100644 --- a/packages/extension-driver-ksqldb/package.json +++ b/packages/extension-driver-ksqldb/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-ksqldb", "description": "ksqlDB driver for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-pg/package.json b/packages/extension-driver-pg/package.json index e86d0ce9..e63854d0 100644 --- a/packages/extension-driver-pg/package.json +++ b/packages/extension-driver-pg/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-pg", "description": "PG driver for Vulcan SQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -24,6 +24,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-redshift/package.json b/packages/extension-driver-redshift/package.json index 1e652faa..4b0ca4dc 100644 --- a/packages/extension-driver-redshift/package.json +++ b/packages/extension-driver-redshift/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-redshift", "description": "Redshift driver for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -27,6 +27,6 @@ "exponential-backoff": "^3.1.1" }, "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-driver-snowflake/package.json b/packages/extension-driver-snowflake/package.json index 4f217baa..cf240fc2 100644 --- a/packages/extension-driver-snowflake/package.json +++ b/packages/extension-driver-snowflake/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-snowflake", "description": "Snowflake driver for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -30,6 +30,6 @@ "@types/snowflake-sdk": "^1.6.8" }, "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-huggingface/package.json b/packages/extension-huggingface/package.json index 4ad69130..cfd612f4 100644 --- a/packages/extension-huggingface/package.json +++ b/packages/extension-huggingface/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-huggingface", "description": "Hugging Face feature for VulcanSQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } } diff --git a/packages/extension-store-canner/package.json b/packages/extension-store-canner/package.json index 58dec1c8..c9a8f583 100644 --- a/packages/extension-store-canner/package.json +++ b/packages/extension-store-canner/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-store-canner", "description": "Canner persistence store for Vulcan SQL", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -24,7 +24,7 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" }, "dependencies": { "@canner/canner-storage": "^0.0.7" diff --git a/packages/serve/package.json b/packages/serve/package.json index cfada274..9b8240ce 100644 --- a/packages/serve/package.json +++ b/packages/serve/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/serve", "description": "VulcanSQL package for serving projects", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -22,7 +22,7 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" }, "dependencies": { "redoc": "2.0.0-rc.76" diff --git a/packages/test-utility/package.json b/packages/test-utility/package.json index cafdf445..e61bc2a6 100644 --- a/packages/test-utility/package.json +++ b/packages/test-utility/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/test-utility", "description": "Vulcan package for extension testing", - "version": "0.10.1", + "version": "0.10.2", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "Apache-2.0", "peerDependencies": { - "@vulcan-sql/core": "~0.10.1-0" + "@vulcan-sql/core": "~0.10.2-0" } }