From 800db008ab4b58d10cfce116b4b976433843e8a7 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 13 Aug 2024 13:21:35 +0100 Subject: [PATCH 1/2] fix(firestore-bigquery-export): fix vulns with extension and script --- firestore-bigquery-export/CHANGELOG.md | 4 + firestore-bigquery-export/extension.yaml | 2 +- .../functions/package-lock.json | 5 +- .../functions/package.json | 2 +- .../scripts/gen-schema-view/package-lock.json | 83 ++- .../scripts/gen-schema-view/package.json | 4 +- .../scripts/import/__tests__/e2e.test.ts | 670 ++++-------------- .../scripts/import/package-lock.json | 138 ++-- .../scripts/import/package.json | 13 +- 9 files changed, 236 insertions(+), 685 deletions(-) diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 7cc441030..0c7d996f1 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.53 + +fixed - bump changetracker to fix npm vulnerabilities + ## Version 0.1.52 fixed - bump changetracker to fix npm vulnerabilities diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index abb52d8ad..78df6b2d6 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.1.52 +version: 0.1.53 specVersion: v1beta displayName: Stream Firestore to BigQuery diff --git a/firestore-bigquery-export/functions/package-lock.json b/firestore-bigquery-export/functions/package-lock.json index 6ec2209ac..ffa69fc87 100644 --- a/firestore-bigquery-export/functions/package-lock.json +++ b/firestore-bigquery-export/functions/package-lock.json @@ -3462,7 +3462,9 @@ "license": "MIT" }, "node_modules/fast-xml-parser": { - "version": "4.3.6", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "funding": [ { "type": "github", @@ -3473,7 +3475,6 @@ "url": "https://paypal.me/naturalintelligence" } ], - "license": "MIT", "optional": true, "dependencies": { "strnum": "^1.0.5" diff --git a/firestore-bigquery-export/functions/package.json b/firestore-bigquery-export/functions/package.json index 47a71fdb0..2f1f10dbc 100644 --- a/firestore-bigquery-export/functions/package.json +++ b/firestore-bigquery-export/functions/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.35", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30", diff --git a/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json b/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json index a240ebdcb..c97850282 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json +++ b/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json @@ -1,12 +1,12 @@ { "name": "@firebaseextensions/fs-bq-schema-views", - "version": "0.4.7", + "version": "0.4.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@firebaseextensions/fs-bq-schema-views", - "version": "0.4.7", + "version": "0.4.8", "license": "Apache-2.0", "dependencies": { "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.33", @@ -41,7 +41,7 @@ }, "../../firestore-bigquery-change-tracker": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "1.1.33", + "version": "1.1.35", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", @@ -852,10 +852,11 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/@grpc/grpc-js": { - "version": "1.10.6", - "license": "Apache-2.0", + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.11.tgz", + "integrity": "sha512-3RaoxOqkHHN2c05bwtBNVJmOf/UwMam0rZYtdl7dsRpsvDwcNpv6LkGgzltQ7xVf822LzBoKEPRvf4D7+xeIDw==", "dependencies": { - "@grpc/proto-loader": "^0.7.10", + "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { @@ -863,12 +864,13 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/@grpc/proto-loader": { - "version": "0.7.12", - "license": "Apache-2.0", + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", + "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", - "protobufjs": "^7.2.4", + "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { @@ -2125,11 +2127,12 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/braces": { - "version": "3.0.2", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, - "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -3089,7 +3092,9 @@ "license": "MIT" }, "../../firestore-bigquery-change-tracker/node_modules/fast-xml-parser": { - "version": "4.3.6", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "funding": [ { "type": "github", @@ -3100,7 +3105,6 @@ "url": "https://paypal.me/naturalintelligence" } ], - "license": "MIT", "optional": true, "dependencies": { "strnum": "^1.0.5" @@ -3145,9 +3149,10 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/fill-range": { - "version": "7.0.1", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, - "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -4011,8 +4016,9 @@ }, "../../firestore-bigquery-change-tracker/node_modules/is-number": { "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, - "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -6501,8 +6507,9 @@ }, "../../firestore-bigquery-change-tracker/node_modules/to-regex-range": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, - "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -7889,12 +7896,12 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.10.6", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.6.tgz", - "integrity": "sha512-xP58G7wDQ4TCmN/cMUHh00DS7SRDv/+lC+xFLrTkMIN8h55X5NhZMLYbvy7dSELP15qlI6hPhNCRWVMtZMwqLA==", + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.11.tgz", + "integrity": "sha512-3RaoxOqkHHN2c05bwtBNVJmOf/UwMam0rZYtdl7dsRpsvDwcNpv6LkGgzltQ7xVf822LzBoKEPRvf4D7+xeIDw==", "optional": true, "dependencies": { - "@grpc/proto-loader": "^0.7.10", + "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { @@ -7902,14 +7909,14 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.12", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.12.tgz", - "integrity": "sha512-DCVwMxqYzpUCiDMl7hQ384FqP4T3DbNpXU8pt681l3UWCip1WUiD5JrkImUwCB9a7f2cq4CUTmi5r/xIMRPY1Q==", + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", + "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", "optional": true, "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", - "protobufjs": "^7.2.4", + "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { @@ -9691,11 +9698,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, - "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -10889,9 +10897,9 @@ "license": "Apache-2.0" }, "node_modules/fast-xml-parser": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", - "integrity": "sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "funding": [ { "type": "github", @@ -10940,9 +10948,10 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, - "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -11937,8 +11946,9 @@ }, "node_modules/is-number": { "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, - "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -15658,8 +15668,9 @@ }, "node_modules/to-regex-range": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, - "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, diff --git a/firestore-bigquery-export/scripts/gen-schema-view/package.json b/firestore-bigquery-export/scripts/gen-schema-view/package.json index d2fe3360e..2585cc971 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/package.json +++ b/firestore-bigquery-export/scripts/gen-schema-view/package.json @@ -1,6 +1,6 @@ { "name": "@firebaseextensions/fs-bq-schema-views", - "version": "0.4.7", + "version": "0.4.8", "description": "Generate strongly-typed BigQuery Views based on raw JSON", "main": "./lib/index.js", "repository": { @@ -31,7 +31,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.33", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36", "@google-cloud/bigquery": "^6.0.3", "commander": "5.0.0", "firebase-admin": "^12.1.0", diff --git a/firestore-bigquery-export/scripts/import/__tests__/e2e.test.ts b/firestore-bigquery-export/scripts/import/__tests__/e2e.test.ts index 04d74e5cf..b722af27f 100644 --- a/firestore-bigquery-export/scripts/import/__tests__/e2e.test.ts +++ b/firestore-bigquery-export/scripts/import/__tests__/e2e.test.ts @@ -1,5 +1,4 @@ import { BigQuery } from "@google-cloud/bigquery"; -import axios from "axios"; import * as childProcess from "child_process"; import * as admin from "firebase-admin"; import * as path from "path"; @@ -11,37 +10,91 @@ const projectId = "extensions-testing"; const bigquery = new BigQuery({ projectId }); -async function runScript(scriptPath: string, callback, args?: string[]) { +async function runScript(scriptPath: string, args?: string[]) { return new Promise((resolve, reject) => { - // keep track of whether callback has been invoked to prevent multiple invocations let invoked = false; const child = childProcess.fork(scriptPath, args, { cwd: __dirname, stdio: [process.stdin, process.stdout, process.stderr, "ipc"], - env: { - ...process.env, - }, + env: { ...process.env }, }); - // listen for errors as they may prevent the exit event from firing child.on("error", (err) => { - if (invoked) return; - invoked = true; - callback(err); - reject(err); + if (!invoked) { + invoked = true; + reject(err); + } }); - // execute the callback once the process has finished running child.on("exit", (code) => { - if (invoked) return; - invoked = true; - const err = code === 0 ? null : new Error("exit code " + code); - callback(err); - resolve(); + if (!invoked) { + invoked = true; + code === 0 ? resolve() : reject(new Error("exit code " + code)); + } }); }); } +async function clearFirestoreData() { + await fetch( + `http://localhost:8080/emulator/v1/projects/extensions-testing/databases/(default)/documents`, + { method: "DELETE" } + ); +} + +async function setupFirestore(collectionName: string) { + process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; + process.env.FIREBASE_CONFIG = JSON.stringify({ + apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", + authDomain: "extensions-testing.firebaseapp.com", + databaseURL: "https://extensions-testing.firebaseio.com", + projectId: "extensions-testing", + storageBucket: "extensions-testing.appspot.com", + messagingSenderId: "219368645393", + appId: "1:219368645393:web:e92083eba0c53f366862b0", + measurementId: "G-QF38ZM1SZN", + }); + + if (!admin.apps.length) { + admin.initializeApp(); + } + + const firestore = admin.firestore(); + await firestore.collection(collectionName).doc("test").set({ test: "test" }); + + return firestore; +} + +async function cleanupBigQuery(datasetName: string) { + await new Promise((resolve) => setTimeout(resolve, 5000)); // Wait for data to settle + const datasetExists = await bigquery.dataset(datasetName).exists(); + if (datasetExists[0]) { + await bigquery.dataset(datasetName).delete({ force: true }); + } +} + +async function runTestScript( + args: string[], + datasetName: string, + tableName: string, + expectedRowCount: number +) { + await runScript(scriptPath, args); + + const [rows] = await repeat( + () => + bigquery + .dataset(datasetName) + .table(`${tableName}_raw_changelog`) + .getRows(), + (rows) => rows[0].length >= expectedRowCount, + 10, + 20000 + ); + + return rows; +} + describe("e2e test CLI", () => { let firestore; let collectionName = "testCollection"; @@ -53,41 +106,12 @@ describe("e2e test CLI", () => { collectionName = `testCollection_${randomID}`; datasetName = `testDataset_${randomID}`; tableName = `testTable_${randomID}`; - - //This is live config, should be emulator? - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore - .collection(collectionName) - .doc("test") - .set({ test: "test" }); + firestore = await setupFirestore(collectionName); }); afterEach(async () => { - // wait 5 seconds for all bigquery data to settle - await new Promise((resolve) => setTimeout(resolve, 5000)); - // if dataset exists, delete it - if ((await bigquery.dataset(datasetName).exists())[0]) { - await bigquery.dataset(datasetName).delete({ force: true }); - } - // delete all data from firestore via axios request - await axios.delete( - `http://localhost:8080/emulator/v1/projects/extensions-testing/databases/(default)/documents` - ); + await cleanupBigQuery(datasetName); + await clearFirestoreData(); }); test(`should import data with old script`, async () => { @@ -110,33 +134,16 @@ describe("e2e test CLI", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 0, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 1); const { operation, - timestamp, document_name, document_id, data, event_id, old_data, + timestamp, } = rows[0]; expect(operation).toBe("IMPORT"); @@ -149,6 +156,7 @@ describe("e2e test CLI", () => { expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); }); + test(`should import data with new script, and add the correct view`, async () => { const args = [ "--non-interactive", @@ -171,34 +179,18 @@ describe("e2e test CLI", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 0, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 1); expect(rows.length).toBeGreaterThanOrEqual(1); + const { operation, - timestamp, document_name, document_id, data, event_id, old_data, + timestamp, } = rows[0]; expect(operation).toBe("IMPORT"); @@ -207,7 +199,6 @@ describe("e2e test CLI", () => { ); expect(document_id).toBe("test"); expect(JSON.parse(data)).toEqual({ test: "test" }); - expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -219,23 +210,14 @@ describe("e2e test CLI", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); test("basic collection group query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore .collection("regions/europe/countries") .doc("france") .set({ name: "France" }); - await firestore .collection("notregions/asia/countries") .doc("japan") @@ -262,39 +244,13 @@ describe("e2e test CLI", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); + const rows = await runTestScript(args, datasetName, tableName, 2); - console.log(datasetName); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 1, - 10, - 20000 - ); - - console.log(rows); expect(rows.length).toBe(2); - const { operation, timestamp, event_id, old_data } = rows[0]; + const { operation, event_id, old_data, timestamp } = rows[0]; expect(operation).toBe("IMPORT"); - // expect(document_name).toBe( - // `projects/extensions-testing/databases/(default)/documents/${collectionName}/test` - // ); - // expect(document_id).toBe("test"); - // expect(JSON.parse(data)).toEqual({ test: "test" }); - expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -306,33 +262,14 @@ describe("e2e test CLI", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); test("wildcarded collection group query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore .collection("regions/europe/countries") .doc("france") .set({ name: "France" }); - await firestore .collection("regions/asia/countries") .doc("japan") @@ -359,45 +296,13 @@ describe("e2e test CLI", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - console.log(datasetName); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 2); expect(rows.length).toBe(2); - const { - operation, - timestamp, - document_name, - document_id, - data, - event_id, - old_data, - } = rows[0]; - expect(operation).toBe("IMPORT"); - // expect(document_name).toBe( - // `projects/extensions-testing/databases/(default)/documents/${collectionName}/test` - // ); - // expect(document_id).toBe("test"); - // expect(JSON.parse(data)).toEqual({ test: "test" }); + const { operation, event_id, old_data, timestamp } = rows[0]; + expect(operation).toBe("IMPORT"); expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -409,37 +314,18 @@ describe("e2e test CLI", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); - test("shouldn't export non-matching results from collection group query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); + test("shouldn't export non-matching results from collection group query", async () => { await firestore .collection("regions/europe/countries") .doc("france") .set({ name: "France" }); - await firestore .collection("regions/asia/countries") .doc("japan") .set({ name: "Japan" }); - await firestore .collection("notregions/asia/countries") .doc("foo") @@ -466,47 +352,13 @@ describe("e2e test CLI", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - //sleep for 2 seconds - await new Promise((resolve) => setTimeout(resolve, 5000)); - - console.log(datasetName); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 2); expect(rows.length).toBe(2); - const { - operation, - timestamp, - document_name, - document_id, - data, - event_id, - old_data, - } = rows[0]; - expect(operation).toBe("IMPORT"); - // expect(document_name).toBe( - // `projects/extensions-testing/databases/(default)/documents/${collectionName}/test` - // ); - // expect(document_id).toBe("test"); - // expect(JSON.parse(data)).toEqual({ test: "test" }); + const { operation, event_id, old_data, timestamp } = rows[0]; + expect(operation).toBe("IMPORT"); expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -518,28 +370,10 @@ describe("e2e test CLI", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); test("should match several wildcards in one query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore .collection("regions/europe/countries/france/cities") .doc("paris") @@ -566,35 +400,15 @@ describe("e2e test CLI", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length >= 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 1); expect(rows.length).toBe(1); - console.log(rows[0]); - const { path_params } = rows[0]; + const { path_params } = rows[0]; const pathParams = JSON.parse(path_params); - expect(pathParams).toHaveProperty("regionId"); - expect(pathParams).toHaveProperty("countryId"); - expect(pathParams.regionId).toBe("europe"); - expect(pathParams.countryId).toBe("france"); + expect(pathParams).toHaveProperty("regionId", "europe"); + expect(pathParams).toHaveProperty("countryId", "france"); }); }); @@ -609,42 +423,12 @@ describe("e2e multi thread", () => { collectionName = `testCollection_${randomID}`; datasetName = `testDataset_${randomID}`; tableName = `testTable_${randomID}`; - - //This is live config, should be emulator? - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore - .collection(collectionName) - .doc("test") - .set({ test: "test" }); + firestore = await setupFirestore(collectionName); }); afterEach(async () => { - // wait 5 seconds for all bigquery data to settle - await new Promise((resolve) => setTimeout(resolve, 5000)); - - // if dataset exists, delete it - if ((await bigquery.dataset(datasetName).exists())[0]) { - await bigquery.dataset(datasetName).delete({ force: true }); - } - // delete all data from firestore via axios request - await axios.delete( - `http://localhost:8080/emulator/v1/projects/extensions-testing/databases/(default)/documents` - ); + await cleanupBigQuery(datasetName); + await clearFirestoreData(); }); test(`should import data with old script`, async () => { @@ -668,33 +452,16 @@ describe("e2e multi thread", () => { "-m", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 0, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 1); const { operation, - timestamp, document_name, document_id, data, event_id, old_data, + timestamp, } = rows[0]; expect(operation).toBe("IMPORT"); @@ -707,6 +474,7 @@ describe("e2e multi thread", () => { expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); }); + test(`should import data with new script, and add the correct view`, async () => { const args = [ "--non-interactive", @@ -730,34 +498,18 @@ describe("e2e multi thread", () => { "-m", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 0, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 1); expect(rows.length).toBeGreaterThanOrEqual(1); + const { operation, - timestamp, document_name, document_id, data, event_id, old_data, + timestamp, } = rows[0]; expect(operation).toBe("IMPORT"); @@ -766,7 +518,6 @@ describe("e2e multi thread", () => { ); expect(document_id).toBe("test"); expect(JSON.parse(data)).toEqual({ test: "test" }); - expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -778,23 +529,14 @@ describe("e2e multi thread", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); test("basic collection group query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore .collection("regions/europe/countries") .doc("france") .set({ name: "France" }); - await firestore .collection("notregions/asia/countries") .doc("japan") @@ -822,39 +564,13 @@ describe("e2e multi thread", () => { "-m", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - console.log(datasetName); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 2); - console.log(rows); expect(rows.length).toBe(2); - const { operation, timestamp, event_id, old_data } = rows[0]; + const { operation, event_id, old_data, timestamp } = rows[0]; expect(operation).toBe("IMPORT"); - // expect(document_name).toBe( - // `projects/extensions-testing/databases/(default)/documents/${collectionName}/test` - // ); - // expect(document_id).toBe("test"); - // expect(JSON.parse(data)).toEqual({ test: "test" }); - expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -866,33 +582,14 @@ describe("e2e multi thread", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); test("wildcarded collection group query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore .collection("regions/europe/countries") .doc("france") .set({ name: "France" }); - await firestore .collection("regions/asia/countries") .doc("japan") @@ -920,45 +617,13 @@ describe("e2e multi thread", () => { "-m", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - console.log(datasetName); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 2); expect(rows.length).toBe(2); - const { - operation, - timestamp, - document_name, - document_id, - data, - event_id, - old_data, - } = rows[0]; - expect(operation).toBe("IMPORT"); - // expect(document_name).toBe( - // `projects/extensions-testing/databases/(default)/documents/${collectionName}/test` - // ); - // expect(document_id).toBe("test"); - // expect(JSON.parse(data)).toEqual({ test: "test" }); + const { operation, event_id, old_data, timestamp } = rows[0]; + expect(operation).toBe("IMPORT"); expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -970,37 +635,18 @@ describe("e2e multi thread", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); - test("shouldn't export non-matching results from collection group query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); + test("shouldn't export non-matching results from collection group query", async () => { await firestore .collection("regions/europe/countries") .doc("france") .set({ name: "France" }); - await firestore .collection("regions/asia/countries") .doc("japan") .set({ name: "Japan" }); - await firestore .collection("notregions/asia/countries") .doc("foo") @@ -1028,47 +674,13 @@ describe("e2e multi thread", () => { "-m", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - //sleep for 2 seconds - await new Promise((resolve) => setTimeout(resolve, 5000)); - - console.log(datasetName); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length > 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 2); expect(rows.length).toBe(2); - const { - operation, - timestamp, - document_name, - document_id, - data, - event_id, - old_data, - } = rows[0]; - expect(operation).toBe("IMPORT"); - // expect(document_name).toBe( - // `projects/extensions-testing/databases/(default)/documents/${collectionName}/test` - // ); - // expect(document_id).toBe("test"); - // expect(JSON.parse(data)).toEqual({ test: "test" }); + const { operation, event_id, old_data, timestamp } = rows[0]; + expect(operation).toBe("IMPORT"); expect(event_id).toBe(""); expect(old_data).toBeNull(); expect(timestamp).toBeDefined(); @@ -1080,28 +692,10 @@ describe("e2e multi thread", () => { const query = view.metadata.view.query; expect(query).toBeDefined(); - const isOldQuery = query.includes("FIRST_VALUE"); - expect(isOldQuery).toBe(false); + expect(query.includes("FIRST_VALUE")).toBe(false); }); test("should match several wildcards in one query", async () => { - process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; - process.env.FIREBASE_CONFIG = JSON.stringify({ - apiKey: "AIzaSyAJTgFI-OVRjgd_10JDWc9T3kxvxY-fUe4", - authDomain: "extensions-testing.firebaseapp.com", - databaseURL: "https://extensions-testing.firebaseio.com", - projectId: "extensions-testing", - storageBucket: "extensions-testing.appspot.com", - messagingSenderId: "219368645393", - appId: "1:219368645393:web:e92083eba0c53f366862b0", - measurementId: "G-QF38ZM1SZN", - }); - // if there is no app, initialize one - if (!admin.apps.length) { - admin.initializeApp(); - } - firestore = admin.firestore(); - await firestore .collection("regions/europe/countries/france/cities") .doc("paris") @@ -1128,34 +722,14 @@ describe("e2e multi thread", () => { "true", ]; - await runScript( - scriptPath, - () => { - console.log("complete!"); - }, - args - ); - - const [rows] = await repeat( - () => - bigquery - .dataset(datasetName) - .table(`${tableName}_raw_changelog`) - .getRows(), - (rows) => rows[0].length >= 1, - 10, - 20000 - ); + const rows = await runTestScript(args, datasetName, tableName, 1); expect(rows.length).toBe(1); - console.log(rows[0]); - const { path_params } = rows[0]; + const { path_params } = rows[0]; const pathParams = JSON.parse(path_params); - expect(pathParams).toHaveProperty("regionId"); - expect(pathParams).toHaveProperty("countryId"); - expect(pathParams.regionId).toBe("europe"); - expect(pathParams.countryId).toBe("france"); + expect(pathParams).toHaveProperty("regionId", "europe"); + expect(pathParams).toHaveProperty("countryId", "france"); }); }); diff --git a/firestore-bigquery-export/scripts/import/package-lock.json b/firestore-bigquery-export/scripts/import/package-lock.json index a1cf9fa69..25f12fcb8 100644 --- a/firestore-bigquery-export/scripts/import/package-lock.json +++ b/firestore-bigquery-export/scripts/import/package-lock.json @@ -1,12 +1,12 @@ { "name": "@firebaseextensions/fs-bq-import-collection", - "version": "0.1.21", + "version": "0.1.22", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@firebaseextensions/fs-bq-import-collection", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.31", @@ -27,7 +27,6 @@ "@types/chai": "^4.2.0", "@types/jest": "29.5.0", "@types/workerpool": "^6.0.0", - "axios": "^1.3.2", "chai": "^4.2.0", "dotenv": "^16.3.1", "jest": "29.5.0", @@ -40,7 +39,7 @@ }, "../../firestore-bigquery-change-tracker": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "1.1.33", + "version": "1.1.35", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", @@ -851,10 +850,11 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/@grpc/grpc-js": { - "version": "1.10.6", - "license": "Apache-2.0", + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.11.tgz", + "integrity": "sha512-3RaoxOqkHHN2c05bwtBNVJmOf/UwMam0rZYtdl7dsRpsvDwcNpv6LkGgzltQ7xVf822LzBoKEPRvf4D7+xeIDw==", "dependencies": { - "@grpc/proto-loader": "^0.7.10", + "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { @@ -862,12 +862,13 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/@grpc/proto-loader": { - "version": "0.7.12", - "license": "Apache-2.0", + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", + "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", - "protobufjs": "^7.2.4", + "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { @@ -2124,11 +2125,12 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/braces": { - "version": "3.0.2", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, - "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -3088,7 +3090,9 @@ "license": "MIT" }, "../../firestore-bigquery-change-tracker/node_modules/fast-xml-parser": { - "version": "4.3.6", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "funding": [ { "type": "github", @@ -3099,7 +3103,6 @@ "url": "https://paypal.me/naturalintelligence" } ], - "license": "MIT", "optional": true, "dependencies": { "strnum": "^1.0.5" @@ -3144,9 +3147,10 @@ } }, "../../firestore-bigquery-change-tracker/node_modules/fill-range": { - "version": "7.0.1", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, - "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -4010,8 +4014,9 @@ }, "../../firestore-bigquery-change-tracker/node_modules/is-number": { "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, - "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -6500,8 +6505,9 @@ }, "../../firestore-bigquery-change-tracker/node_modules/to-regex-range": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, - "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -7775,12 +7781,12 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.10.6", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.6.tgz", - "integrity": "sha512-xP58G7wDQ4TCmN/cMUHh00DS7SRDv/+lC+xFLrTkMIN8h55X5NhZMLYbvy7dSELP15qlI6hPhNCRWVMtZMwqLA==", + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.11.tgz", + "integrity": "sha512-3RaoxOqkHHN2c05bwtBNVJmOf/UwMam0rZYtdl7dsRpsvDwcNpv6LkGgzltQ7xVf822LzBoKEPRvf4D7+xeIDw==", "optional": true, "dependencies": { - "@grpc/proto-loader": "^0.7.10", + "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" }, "engines": { @@ -7788,14 +7794,14 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.12", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.12.tgz", - "integrity": "sha512-DCVwMxqYzpUCiDMl7hQ384FqP4T3DbNpXU8pt681l3UWCip1WUiD5JrkImUwCB9a7f2cq4CUTmi5r/xIMRPY1Q==", + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", + "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", "optional": true, "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", - "protobufjs": "^7.2.4", + "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { @@ -8831,18 +8837,8 @@ }, "node_modules/asynckit": { "version": "0.4.0", - "devOptional": true, - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.6.0", - "dev": true, "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } + "optional": true }, "node_modules/babel-jest": { "version": "29.7.0", @@ -9061,11 +9057,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, - "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -9368,8 +9365,8 @@ }, "node_modules/combined-stream": { "version": "1.0.8", - "devOptional": true, "license": "MIT", + "optional": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -9594,8 +9591,8 @@ }, "node_modules/delayed-stream": { "version": "1.0.0", - "devOptional": true, "license": "MIT", + "optional": true, "engines": { "node": ">=0.4.0" } @@ -9962,9 +9959,9 @@ "license": "Apache-2.0" }, "node_modules/fast-xml-parser": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", - "integrity": "sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "funding": [ { "type": "github", @@ -10035,9 +10032,10 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, - "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -10145,39 +10143,6 @@ "@types/serve-static": "*" } }, - "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/forwarded": { "version": "0.2.0", "license": "MIT", @@ -10895,8 +10860,9 @@ }, "node_modules/is-number": { "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, - "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -12878,11 +12844,6 @@ "node": ">= 0.10" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "dev": true, - "license": "MIT" - }, "node_modules/pseudomap": { "version": "1.0.2", "license": "ISC" @@ -13565,8 +13526,9 @@ }, "node_modules/to-regex-range": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, - "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, diff --git a/firestore-bigquery-export/scripts/import/package.json b/firestore-bigquery-export/scripts/import/package.json index 3895d87c8..bb3abe2f6 100644 --- a/firestore-bigquery-export/scripts/import/package.json +++ b/firestore-bigquery-export/scripts/import/package.json @@ -1,6 +1,6 @@ { "name": "@firebaseextensions/fs-bq-import-collection", - "version": "0.1.21", + "version": "0.1.22", "description": "Import a Firestore Collection into a BigQuery Changelog Table", "main": "./lib/index.js", "repository": { @@ -26,7 +26,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.31", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36", "@google-cloud/bigquery": "^5.6.0", "commander": "5.0.0", "filenamify": "^4.2.0", @@ -41,14 +41,13 @@ "@types/chai": "^4.2.0", "@types/jest": "29.5.0", "@types/workerpool": "^6.0.0", - "axios": "^1.3.2", "chai": "^4.2.0", "dotenv": "^16.3.1", + "jest": "29.5.0", "nanoid": "^4.0.0", "rimraf": "^2.6.3", + "ts-jest": "29.1.2", "ts-node": "^10.9.1", - "typescript": "^4.2.4", - "jest": "29.5.0", - "ts-jest": "29.1.2" + "typescript": "^4.2.4" } -} \ No newline at end of file +} From 0a1a5e9b7b489a886ccfc2464a903ea73daaa5d3 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Thu, 15 Aug 2024 15:07:46 +0100 Subject: [PATCH 2/2] chore(firestore-bigquery-export): update lockfiles --- .../functions/__tests__/e2e.test.ts | 11 +++++++---- firestore-bigquery-export/functions/package-lock.json | 8 ++++---- .../scripts/gen-schema-view/package-lock.json | 4 ++-- .../scripts/import/package-lock.json | 4 ++-- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/firestore-bigquery-export/functions/__tests__/e2e.test.ts b/firestore-bigquery-export/functions/__tests__/e2e.test.ts index c2d95cbcb..4b6ff773e 100644 --- a/firestore-bigquery-export/functions/__tests__/e2e.test.ts +++ b/firestore-bigquery-export/functions/__tests__/e2e.test.ts @@ -2,13 +2,16 @@ import * as admin from "firebase-admin"; import { BigQuery } from "@google-cloud/bigquery"; /** Set defaults */ -const bqProjectId = "extensions-testing"; +const bqProjectId = "dev-extensions-testing"; const datasetId = "firestore_export"; const tableId = "bq_e2e_test_raw_changelog"; /** Init resources */ admin.initializeApp({ projectId: bqProjectId }); -const bq = new BigQuery({ projectId: "extensions-testing" }); +const bq = new BigQuery({ + projectId: "dev-extensions-testing", + // location: "us-central1", +}); import { documentData } from "./fixtures/documentData"; /*** @@ -27,7 +30,7 @@ describe("e2e", () => { const docRef = await db.collection("posts").add(testData); /** Wait for 20 seconds */ - await new Promise((resolve) => setTimeout(resolve, 5000)); + await new Promise((resolve) => setTimeout(resolve, 20000)); /** Get the latest record from this table */ const [changeLogQuery] = await bq.createQueryJob({ @@ -57,5 +60,5 @@ describe("e2e", () => { expect(result.singleReference).toBe("reference/reference1"); expect(result.reference_list[0]).toBe("reference/reference1"); expect(result.reference_list[1]).toBe("reference/reference2"); - }, 10000); + }, 30000); }); diff --git a/firestore-bigquery-export/functions/package-lock.json b/firestore-bigquery-export/functions/package-lock.json index ffa69fc87..1414b7d71 100644 --- a/firestore-bigquery-export/functions/package-lock.json +++ b/firestore-bigquery-export/functions/package-lock.json @@ -7,7 +7,7 @@ "name": "firestore-bigquery-export", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.35", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30", @@ -572,9 +572,9 @@ } }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker": { - "version": "1.1.35", - "resolved": "https://registry.npmjs.org/@firebaseextensions/firestore-bigquery-change-tracker/-/firestore-bigquery-change-tracker-1.1.35.tgz", - "integrity": "sha512-LbVwBsqZkyPAuX0cjhpCjL8b4TrNlFlUGjnOzkq1xX1hBDLK+ekMK/nn3sZD7e2gunKDEKkXs9CKX2cvJMMO0Q==", + "version": "1.1.36", + "resolved": "https://registry.npmjs.org/@firebaseextensions/firestore-bigquery-change-tracker/-/firestore-bigquery-change-tracker-1.1.36.tgz", + "integrity": "sha512-JQlcnREreodtZoBz5lNt0nT1VTpItC9W+m15sJh2Q3elFWafhs1K172ujdc9A7WZkgqR6WJ8+1NaO4oDnnNnHw==", "dependencies": { "@google-cloud/bigquery": "^7.6.0", "@google-cloud/resource-manager": "^5.1.0", diff --git a/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json b/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json index c97850282..c789a1f18 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json +++ b/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json @@ -9,7 +9,7 @@ "version": "0.4.8", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.33", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36", "@google-cloud/bigquery": "^6.0.3", "commander": "5.0.0", "firebase-admin": "^12.1.0", @@ -41,7 +41,7 @@ }, "../../firestore-bigquery-change-tracker": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "1.1.35", + "version": "1.1.36", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", diff --git a/firestore-bigquery-export/scripts/import/package-lock.json b/firestore-bigquery-export/scripts/import/package-lock.json index 25f12fcb8..e0866447d 100644 --- a/firestore-bigquery-export/scripts/import/package-lock.json +++ b/firestore-bigquery-export/scripts/import/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.31", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36", "@google-cloud/bigquery": "^5.6.0", "commander": "5.0.0", "filenamify": "^4.2.0", @@ -39,7 +39,7 @@ }, "../../firestore-bigquery-change-tracker": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "1.1.35", + "version": "1.1.36", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0",