From a5e53717b844fcda40636de1313454df77a6d296 Mon Sep 17 00:00:00 2001 From: Martin Lysk Date: Wed, 9 Oct 2024 14:07:42 +0200 Subject: [PATCH 1/2] WIP first steps with benchmarking --- lix/packages/sdk/package.json | 1 + .../src/benchmark/changetable-inlang.bench.ts | 166 ++++++++++ .../src/benchmark/changetable-sketch.bench.ts | 300 ++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100644 lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts create mode 100644 lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts diff --git a/lix/packages/sdk/package.json b/lix/packages/sdk/package.json index 18ff4b83c3..b46f0ad2be 100644 --- a/lix/packages/sdk/package.json +++ b/lix/packages/sdk/package.json @@ -8,6 +8,7 @@ }, "scripts": { "build": "tsc --build", + "bench": "vitest bench", "test": "tsc --noEmit && vitest run --coverage", "test:watch": "vitest", "dev": "tsc --watch", diff --git a/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts b/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts new file mode 100644 index 0000000000..2bae28f1fa --- /dev/null +++ b/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts @@ -0,0 +1,166 @@ +import { v4 } from "uuid"; +import { afterEach, beforeEach, bench, describe } from "vitest"; +import { + getLeafChange, + newLixFile, + openLixInMemory, + type Change, +} from "../index.js"; + +const createChange = ( + type: "bundle" | "message" | "variant", + payload: any, + parentChangeId?: string, +) => { + const change: Change = { + id: v4(), + parent_id: parentChangeId, + operation: "create", + file_id: "mock", + plugin_key: "inlang", + type: type, + // @ts-expect-error - type error in lix + value: JSON.stringify(payload[type]), + }; + return change; +}; + +const setupLix = async (nMessages: number) => { + const lix = await openLixInMemory({ + blob: await newLixFile(), + }); + + const mockChanges: Change[] = []; + + for (let i = 0; i < nMessages; i++) { + const payloads = getPayloadsForId(v4()); + + // lets asume we create a bundle once and don't change it to much over its lifetime + createChange("bundle", payloads); + + // lets asume we add 4 languages to the message and don't change it to much + // -> 6 changes + let lastMessageChangeId: undefined | string = undefined; + for (let i = 0; i < 6; i++) { + const change = createChange("message", payloads, lastMessageChangeId); + mockChanges.push(change) + lastMessageChangeId = change.id + } + + // lets assume imports some variants once (1) change and others get edited quit heavly 120 characters 10 times rewriten (1200 changes) + // -> 100 changes per variant + createChange("variant", payloads); + let lastVariantChangeId: undefined | string = undefined; + for (let i = 0; i < 100; i++) { + const change = createChange("variant", payloads, lastVariantChangeId); + mockChanges.push(change) + lastVariantChangeId = change.id + } + } + + const firstChangeId = mockChanges[0]!.id + + const batchSize = 256; + // Insert changes in batches + for (let i = 0; i < mockChanges.length; i += batchSize) { + const batch = mockChanges.slice(i, i + batchSize); + await lix.db.insertInto("change").values(batch).executeTakeFirst(); + } + // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + + return { lix, firstChangeId }; +}; + +async function wait(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); // 1000 milliseconds = 1 second + }); +} + +for (let i = 0; i < 4; i++) { + const nMessages = Math.pow(10, i); + describe( + "select changes via on " + + nMessages + + " messages -> " + nMessages * (1+6+100) + " changes", + async () => { + let project = await setupLix(nMessages); + + bench("payload->property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("resizingConstraint"), + "=", + 100, + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("plugin_key", "=", "mock1") + + .executeTakeFirst(); + + // console.log(result) + }); + + bench( + "getLeafNode", + async () => { + await getLeafChange({ + lix: project.lix, + change: { id: project.firstChangeId } as Change, + }); + + // console.log(result) + }, + { + iterations: 2, + }, + ); + }, + ); +} + +const getPayloadsForId = (id: string) => { + return { + bundle: { + id: id, + declarations: [ + { + type: "input-variable", + name: "name", + }, + ], + }, + message: { + id: id + "_en", + bundleId: id, + locale: "en", + selectors: [], + }, + variant: { + id: id + "_en_variant_one", + messageId: "depressed_dog_en", + matches: [], + pattern: [ + { type: "text", value: "Good morning " }, + { + type: "expression", + arg: { type: "variable-reference", name: "name" }, + }, + { type: "text", value: "!" }, + ], + }, + }; +}; diff --git a/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts b/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts new file mode 100644 index 0000000000..b62d23d2d7 --- /dev/null +++ b/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts @@ -0,0 +1,300 @@ +import { v4 } from "uuid"; +import { afterEach, beforeEach, bench, describe } from "vitest"; +import { + getLeafChange, + newLixFile, + openLixInMemory, + type Change, +} from "../index.js"; + +const createChange = (parentChangeId?: string) => { + samplePaylod.resizingConstraint += 1; + + const change: Change = { + id: v4(), + parent_id: parentChangeId, + operation: "create", + file_id: "mock", + plugin_key: parentChangeId ? "mock" : "mock1", + type: "mock", + // @ts-expect-error - type error in lix + value: JSON.stringify(samplePaylod), + }; + return change; +}; + +const setupLix = async (nChanges: number) => { + const lix = await openLixInMemory({ + blob: await newLixFile(), + }); + + const mockChanges: Change[] = []; + let lastChangeId: undefined | string = undefined; + let firstChangeId: undefined | string = undefined; + for (let i = 0; i < nChanges; i++) { + const change = createChange(lastChangeId); + lastChangeId = change.id; + if (!firstChangeId) { + firstChangeId = change.id; + } + mockChanges.push(change); + } + + const batchSize = 256; + // Insert changes in batches + for (let i = 0; i < mockChanges.length; i += batchSize) { + const batch = mockChanges.slice(i, i + batchSize); + await lix.db.insertInto("change").values(batch).executeTakeFirst(); + } + // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + + return { lix, firstChangeId }; +}; + +async function wait(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); // 1000 milliseconds = 1 second + }); +} + +for (let i = 0; i < 5; i++) { + const nChanges = Math.pow(10, i); + describe( + "select changes via property `where value->>resizingConstraint = 100` on " + + nChanges + + " changes", + async () => { + let project = await setupLix(nChanges); + + bench.skip("payload->property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("resizingConstraint"), + "=", + 100, + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench.skip("column", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("plugin_key", "=", "mock1") + + .executeTakeFirst(); + + // console.log(result) + }); + + bench.skip( + "getLeafNode", + async () => { + await getLeafChange({ + lix: project.lix, + change: { id: project.firstChangeId } as Change, + }); + + // console.log(result) + }, + { + iterations: 2, + }, + ); + }, + ); +} + +const samplePaylod = { + _class: "rectangle", + do_objectID: "AEB63D36-FA90-4F68-90B0-BFEC297D1AF0", + booleanOperation: -1, + isFixedToViewport: false, + isFlippedHorizontal: false, + isFlippedVertical: false, + isLocked: false, + isTemplate: false, + isVisible: true, + layerListExpandedType: 0, + name: "Screenshot", + nameIsFixed: true, + resizingConstraint: 63, + resizingType: 0, + rotation: 0, + shouldBreakMaskChain: false, + exportOptions: { + _class: "exportOptions", + includedLayerIds: [], + layerOptions: 0, + shouldTrim: false, + exportFormats: [], + }, + frame: { + _class: "rect", + constrainProportions: false, + height: 848, + width: 1280, + x: 0, + y: 79, + }, + clippingMaskMode: 0, + hasClippingMask: false, + style: { + _class: "style", + do_objectID: "E52F9334-AB26-4976-95D6-7B07D3B76545", + endMarkerType: 0, + miterLimit: 10, + startMarkerType: 0, + windingRule: 1, + blur: { + _class: "blur", + isEnabled: false, + center: "{0.5, 0.5}", + motionAngle: 0, + radius: 10, + saturation: 1, + type: 0, + }, + borderOptions: { + _class: "borderOptions", + isEnabled: true, + dashPattern: [], + lineCapStyle: 0, + lineJoinStyle: 0, + }, + borders: [], + colorControls: { + _class: "colorControls", + isEnabled: false, + brightness: 0, + contrast: 1, + hue: 0, + saturation: 1, + }, + contextSettings: { + _class: "graphicsContextSettings", + blendMode: 0, + opacity: 1, + }, + fills: [ + { + _class: "fill", + isEnabled: true, + fillType: 4, + color: { + _class: "color", + alpha: 1, + blue: 0, + green: 0, + red: 1, + }, + contextSettings: { + _class: "graphicsContextSettings", + blendMode: 0, + opacity: 1, + }, + gradient: { + _class: "gradient", + elipseLength: 0, + from: "{0.5, 0}", + gradientType: 0, + to: "{0.5, 1}", + stops: [ + { + _class: "gradientStop", + position: 0, + color: { + _class: "color", + alpha: 1, + blue: 1, + green: 1, + red: 1, + }, + }, + { + _class: "gradientStop", + position: 1, + color: { + _class: "color", + alpha: 1, + blue: 0, + green: 0, + red: 0, + }, + }, + ], + }, + image: { + _class: "MSJSONFileReference", + _ref_class: "MSImageData", + _ref: "images/f0e9c250506c97c18d1766a00a7438a22dbbc96b.png", + }, + noiseIndex: 0, + noiseIntensity: 0, + patternFillType: 1, + patternTileScale: 1, + }, + ], + innerShadows: [], + shadows: [], + }, + edited: false, + isClosed: true, + pointRadiusBehaviour: 1, + points: [ + { + _class: "curvePoint", + cornerRadius: 0, + cornerStyle: 0, + curveFrom: "{0, 0}", + curveMode: 1, + curveTo: "{0, 0}", + hasCurveFrom: false, + hasCurveTo: false, + point: "{0, 0}", + }, + { + _class: "curvePoint", + cornerRadius: 0, + cornerStyle: 0, + curveFrom: "{1, 0}", + curveMode: 1, + curveTo: "{1, 0}", + hasCurveFrom: false, + hasCurveTo: false, + point: "{1, 0}", + }, + { + _class: "curvePoint", + cornerRadius: 10, + cornerStyle: 0, + curveFrom: "{1, 1}", + curveMode: 1, + curveTo: "{1, 1}", + hasCurveFrom: false, + hasCurveTo: false, + point: "{1, 1}", + }, + { + _class: "curvePoint", + cornerRadius: 10, + cornerStyle: 0, + curveFrom: "{0, 1}", + curveMode: 1, + curveTo: "{0, 1}", + hasCurveFrom: false, + hasCurveTo: false, + point: "{0, 1}", + }, + ], + fixedRadius: 0, + needsConvertionToNewRoundCorners: false, + hasConvertedToNewRoundCorners: true, +}; From 23d721a667cec4964b812311461c98aaaac30297 Mon Sep 17 00:00:00 2001 From: Martin Lysk Date: Mon, 14 Oct 2024 10:20:27 +0200 Subject: [PATCH 2/2] WIP more benchmarks added, addes recursive query --- .../src/benchmark/changetable-inlang.bench.ts | 65 +++++- .../changetable-nonoverflowing.bench.ts | 204 ++++++++++++++++++ .../changetable-overflowing.bench.ts | 204 ++++++++++++++++++ .../src/benchmark/changetable-sketch.bench.ts | 111 ++++++++-- .../changetable-subsetresult.bench.ts | 204 ++++++++++++++++++ lix/packages/sdk/src/database/createSchema.ts | 3 + lix/packages/sdk/src/database/schema.ts | 2 + .../src/query-utilities/get-leaf-change.ts | 46 ++++ lix/packages/sdk/src/types.ts | 2 +- 9 files changed, 814 insertions(+), 27 deletions(-) create mode 100644 lix/packages/sdk/src/benchmark/changetable-nonoverflowing.bench.ts create mode 100644 lix/packages/sdk/src/benchmark/changetable-overflowing.bench.ts create mode 100644 lix/packages/sdk/src/benchmark/changetable-subsetresult.bench.ts diff --git a/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts b/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts index 2bae28f1fa..9b21a283aa 100644 --- a/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts +++ b/lix/packages/sdk/src/benchmark/changetable-inlang.bench.ts @@ -1,11 +1,13 @@ import { v4 } from "uuid"; -import { afterEach, beforeEach, bench, describe } from "vitest"; +import { afterEach, beforeEach, bench, describe, expect } from "vitest"; import { getLeafChange, newLixFile, openLixInMemory, type Change, } from "../index.js"; +import { getLeafChangeRecursive } from "../query-utilities/get-leaf-change.js"; +import { sql } from "kysely"; const createChange = ( type: "bundle" | "message" | "variant", @@ -21,11 +23,14 @@ const createChange = ( type: type, // @ts-expect-error - type error in lix value: JSON.stringify(payload[type]), + valueB: sql`cast (${JSON.stringify(payload[type])} as jsonb)` + }; return change; }; const setupLix = async (nMessages: number) => { + console.log('setting up lix with ' + nMessages) const lix = await openLixInMemory({ blob: await newLixFile(), }); @@ -36,7 +41,7 @@ const setupLix = async (nMessages: number) => { const payloads = getPayloadsForId(v4()); // lets asume we create a bundle once and don't change it to much over its lifetime - createChange("bundle", payloads); + mockChanges.push(createChange("bundle", payloads)); // lets asume we add 4 languages to the message and don't change it to much // -> 6 changes @@ -67,6 +72,7 @@ const setupLix = async (nMessages: number) => { await lix.db.insertInto("change").values(batch).executeTakeFirst(); } // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + console.log('setting up lix with ' + nMessages + '.... done') return { lix, firstChangeId }; }; @@ -79,7 +85,7 @@ async function wait(ms: number): Promise { }); } -for (let i = 0; i < 4; i++) { +for (let i = 0; i < 5; i++) { const nMessages = Math.pow(10, i); describe( "select changes via on " + @@ -88,12 +94,27 @@ for (let i = 0; i < 4; i++) { async () => { let project = await setupLix(nMessages); - bench("payload->property", async () => { + bench("payload JSON ->property", async () => { let result = await project.lix.db .selectFrom("change") .selectAll() .where( - (eb) => eb.ref("value", "->>").key("resizingConstraint"), + (eb) => eb.ref("value", "->>").key("id"), + "=", + 100, + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("payload JSON B ->property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("id"), "=", 100, ) @@ -113,7 +134,17 @@ for (let i = 0; i < 4; i++) { // console.log(result) }); + } + ); +} +for (let i = 0; i < 4; i++) { + const nMessages = Math.pow(10, i); + describe( + "get Leaf Node " + + nMessages + + " messages -> " + nMessages * (1+6+100) + " changes", + async () => { bench( "getLeafNode", async () => { @@ -128,6 +159,30 @@ for (let i = 0; i < 4; i++) { iterations: 2, }, ); + + bench( + "getLeafNodeRecursive", + async () => { + + + const result = await getLeafChangeRecursive({ + lix: project.lix, + change: { id: project.firstChangeId } as Change, + }); + + // const result2 = await getLeafChange({ + // lix: project.lix, + // change: { id: project.firstChangeId } as Change, + // }); + + // expect(result).toStrictEqual(result2) + + // console.log('RESULTS:', result, result2) + + + // console.log(result) + } + ); }, ); } diff --git a/lix/packages/sdk/src/benchmark/changetable-nonoverflowing.bench.ts b/lix/packages/sdk/src/benchmark/changetable-nonoverflowing.bench.ts new file mode 100644 index 0000000000..8ef2b3027a --- /dev/null +++ b/lix/packages/sdk/src/benchmark/changetable-nonoverflowing.bench.ts @@ -0,0 +1,204 @@ +import { v4 } from "uuid"; +import { afterEach, beforeEach, bench, describe } from "vitest"; +import { + getLeafChange, + newLixFile, + openLixInMemory, + type Change, +} from "../index.js"; +import { getLeafChangeRecursive } from "../query-utilities/get-leaf-change.js"; +import { sql, type RawBuilder } from "kysely"; +import fs from 'fs' + +function json (object: T): RawBuilder { + return sql`jsonb(${JSON.stringify(object)})` + } + +const createChange = (parentChangeId?: string) => { + + const change: Change = { + id: v4(), + parent_id: parentChangeId, + operation: "create", + file_id: "mock", + plugin_key: parentChangeId ? "mock" : "mock1", + type: "mock", + // @ts-expect-error - type error in lix + value: JSON.stringify(samplePaylod), + valueB: json(samplePaylod) as any, + commit_id: 'abuse_for_column_in_overflow' + }; + return change; +}; + +const setupLix = async (nChanges: number) => { + console.log('setting up lix for ' + nChanges + ' changes' ) + const lix = await openLixInMemory({ + blob: await newLixFile(), + }); + + await sql`PRAGMA page_size = 16384;`.execute(lix.db) + + const mockChanges: Change[] = []; + let lastChangeId: undefined | string = undefined; + let firstChangeId: undefined | string = undefined; + for (let i = 0; i < nChanges; i++) { + const change = createChange(lastChangeId); + lastChangeId = change.id; + if (!firstChangeId) { + firstChangeId = change.id; + } + mockChanges.push(change); + } + + const batchSize = 1; + // Insert changes in batches + for (let i = 0; i < mockChanges.length; i += batchSize) { + const batch = mockChanges.slice(i, i + batchSize); + + // console.log(lix.db.insertInto("change").values(batch).compile()) + await lix.db.insertInto("change").values(batch).executeTakeFirst(); + } + // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + console.log('setting up lix for ' + nChanges + ' changes.... done' ) + + + const dbContent = await lix.toBlob() + const ab = await dbContent.arrayBuffer() + const buffer = Buffer.from(ab) + + fs.writeFile('output.db', buffer, (err) => { + if (err) throw err; + console.log('Blob has been written to output.txt'); + }); + + return { lix, firstChangeId }; +}; + +async function wait(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); // 1000 milliseconds = 1 second + }); +} + +for (let i = 0; i < 5; i++) { + const nChanges = Math.pow(10, i); + describe( + "select changes via JSON / JSONB / COLUMN (last and first) " + + nChanges + + " changes", + async () => { + let project = await setupLix(nChanges); + + bench("JSON payload-> first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("a"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + + // console.log(result) + }); + + + + bench("JSON payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("n"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + + bench("JSONB payload->first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("a"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("JSONB payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("n"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("plugin_key", "=", "mock1") + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column in overflow", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("commit_id", "=", "abuse_for_column_in_overflow") + + .executeTakeFirst(); + + // console.log(result) + }); + + + } + ); +} + +const samplePaylod = { + "z": "1", + "a": "mock1", + "b": "1", + "c": "1", + "d": "1", + "e": "1", + "f": "1", + "g": "1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz 18abcdefghijklmnopqrstuvwxyz 19abcdefghijklmnopqrstuvwxyz 20abcdefghijklmnopqrstuvwxyz 21abcdefghijklmnopqrstuvwxyz 22abcdefghijklmnopqrstuvwxyz 23abcdefghijklmnopqrstuvwxyz 24abcdefghijklmnopqrstuvwxyz 25abcdefghijklmnopqrstuvwxyz 26abcdefghijklmnopqrstuvwxyz 27abcdefghijklmnopqrstuvwxyz 28abcdefghijklmnopqrstuvwxyz 29abcdefghijklmnopqrstuvwxyz 30abcdefghijklmnopqrstuvwxyz 31abcdefghijklmnopqrstuvwxyz 32abcdefghijklmnopqrstuvwxyz 33abcdefghijklmnopqrstuvwxyz 34abcdefghijklmnopqrstuvwxyz 35abcdefghijklmnopqrstuvwxyz 36abcdefghijklmnopqrstuvwxyz 37abcdefghijklmnopqrstuvwxyz 38abcdefghijklmnopqrstuvwxyz 39abcdefghijklmnopqrstuvwxyz 40abcdefghijklmnopqrstuvwxyz 41abcdefghijklmnopqrstuvwxyz 42abcdefghijklmnopqrstuvwxyz 43abcdefghijklmnopqrstuvwxyz 44abcdefghijklmnopqrstuvwxyz 45abcdefghijklmnopqrstuvwxyz 46abcdefghijklmnopqrstuvwxyz 47abcdefghijklmnopqrstuvwxyz 48abcdefghijklmnopqrstuvwxyz 49abcdefghijklmnopqrstuvwxyz 50abcdefghijklmnopqrstuvwxyz 51abcdefghijklmnopqrstuvwxyz 52abcdefghijklmnopqrstuvwxyz 53abcdefghijklmnopqrstuvwxyz 54abcdefghijklmnopqrstuvwxyz 55abcdefghijklmnopqrstuvwxyz 56abcdefghijklmnopqrstuvwxyz 57abcdefghijklmnopqrstuvwxyz 58abcdefghijklmnopqrstuvwxyz 59abcdefghijklmnopqrstuvwxyz 60abcdefghijklmnopqrstuvwxyz 61abcdefghijklmnopqrstuvwxyz 62abcdefghijklmnopqrstuvwxyz abcdefghijklmnopqrstuvwxyz 63 abcdefghijklmnopqrstuvwxyz -------------------- 1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz 18abcdefghijklmnopqrstuvwxyz 19abcdefghijklmnopqrstuvwxyz 20abcdefghijklmnopqrstuvwxyz 21abcdefghijklmnopqrstuvwxyz 22abcdefghijklmnopqrstuvwxyz 23abcdefghijklmnopqrstuvwxyz 24abcdefghijklmnopqrstuvwxyz 25abcdefghijklmnopqrstuvwxyz 26abcdefghijklmnopqrstuvwxyz 27abcdefghijklmnopqrstuvwxyz 28abcdefghijklmnopqrstuvwxyz 29abcdefghijklmnopqrstuvwxyz 30abcdefghijklmnopqrstuvwxyz 31abcdefghijklmnopqrstuvwxyz 32abcdefghijklmnopqrstuvwxyz 0123456789a0123456789a0123456789a", + "h": "I AM STILL THE SAME", + "i": "0123456789a0123456789a0123456789a", + "j": "0123456789a0123456789a0123456789a", + "k": "0123456789a0123456789a0123456789a", + "l": "0123456789a0123456789a0123456789a", + "m": "1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz ", + "n": "mock1", +} + diff --git a/lix/packages/sdk/src/benchmark/changetable-overflowing.bench.ts b/lix/packages/sdk/src/benchmark/changetable-overflowing.bench.ts new file mode 100644 index 0000000000..a2eae78f4b --- /dev/null +++ b/lix/packages/sdk/src/benchmark/changetable-overflowing.bench.ts @@ -0,0 +1,204 @@ +import { v4 } from "uuid"; +import { afterEach, beforeEach, bench, describe } from "vitest"; +import { + getLeafChange, + newLixFile, + openLixInMemory, + type Change, +} from "../index.js"; +import { getLeafChangeRecursive } from "../query-utilities/get-leaf-change.js"; +import { sql, type RawBuilder } from "kysely"; +import fs from 'fs' + +function json (object: T): RawBuilder { + return sql`jsonb(${JSON.stringify(object)})` + } + +const createChange = (parentChangeId?: string) => { + + const change: Change = { + id: v4(), + parent_id: parentChangeId, + operation: "create", + file_id: "mock", + plugin_key: parentChangeId ? "mock" : "mock1", + type: "mock", + // @ts-expect-error - type error in lix + value: JSON.stringify(samplePaylod), + valueB: json(samplePaylod) as any, + commit_id: 'abuse_for_column_in_overflow' + }; + return change; +}; + +const setupLix = async (nChanges: number) => { + console.log('setting up lix for ' + nChanges + ' changes' ) + const lix = await openLixInMemory({ + blob: await newLixFile(), + }); + + await sql`PRAGMA page_size = 512;`.execute(lix.db) + + const mockChanges: Change[] = []; + let lastChangeId: undefined | string = undefined; + let firstChangeId: undefined | string = undefined; + for (let i = 0; i < nChanges; i++) { + const change = createChange(lastChangeId); + lastChangeId = change.id; + if (!firstChangeId) { + firstChangeId = change.id; + } + mockChanges.push(change); + } + + const batchSize = 1; + // Insert changes in batches + for (let i = 0; i < mockChanges.length; i += batchSize) { + const batch = mockChanges.slice(i, i + batchSize); + + // console.log(lix.db.insertInto("change").values(batch).compile()) + await lix.db.insertInto("change").values(batch).executeTakeFirst(); + } + // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + console.log('setting up lix for ' + nChanges + ' changes.... done' ) + + + const dbContent = await lix.toBlob() + const ab = await dbContent.arrayBuffer() + const buffer = Buffer.from(ab) + + fs.writeFile('output.db', buffer, (err) => { + if (err) throw err; + console.log('Blob has been written to output.txt'); + }); + + return { lix, firstChangeId }; +}; + +async function wait(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); // 1000 milliseconds = 1 second + }); +} + +for (let i = 0; i < 5; i++) { + const nChanges = Math.pow(10, i); + describe( + "select changes via JSON / JSONB / COLUMN (last and first) " + + nChanges + + " changes", + async () => { + let project = await setupLix(nChanges); + + bench("JSON payload-> first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("a"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + + // console.log(result) + }); + + + + bench("JSON payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("n"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + + bench("JSONB payload->first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("a"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("JSONB payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("n"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("plugin_key", "=", "mock1") + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column in overflow", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("commit_id", "=", "abuse_for_column_in_overflow") + + .executeTakeFirst(); + + // console.log(result) + }); + + + } + ); +} + +const samplePaylod = { + "z": "1", + "a": "mock1", + "b": "1", + "c": "1", + "d": "1", + "e": "1", + "f": "1", + "g": "1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz 18abcdefghijklmnopqrstuvwxyz 19abcdefghijklmnopqrstuvwxyz 20abcdefghijklmnopqrstuvwxyz 21abcdefghijklmnopqrstuvwxyz 22abcdefghijklmnopqrstuvwxyz 23abcdefghijklmnopqrstuvwxyz 24abcdefghijklmnopqrstuvwxyz 25abcdefghijklmnopqrstuvwxyz 26abcdefghijklmnopqrstuvwxyz 27abcdefghijklmnopqrstuvwxyz 28abcdefghijklmnopqrstuvwxyz 29abcdefghijklmnopqrstuvwxyz 30abcdefghijklmnopqrstuvwxyz 31abcdefghijklmnopqrstuvwxyz 32abcdefghijklmnopqrstuvwxyz 33abcdefghijklmnopqrstuvwxyz 34abcdefghijklmnopqrstuvwxyz 35abcdefghijklmnopqrstuvwxyz 36abcdefghijklmnopqrstuvwxyz 37abcdefghijklmnopqrstuvwxyz 38abcdefghijklmnopqrstuvwxyz 39abcdefghijklmnopqrstuvwxyz 40abcdefghijklmnopqrstuvwxyz 41abcdefghijklmnopqrstuvwxyz 42abcdefghijklmnopqrstuvwxyz 43abcdefghijklmnopqrstuvwxyz 44abcdefghijklmnopqrstuvwxyz 45abcdefghijklmnopqrstuvwxyz 46abcdefghijklmnopqrstuvwxyz 47abcdefghijklmnopqrstuvwxyz 48abcdefghijklmnopqrstuvwxyz 49abcdefghijklmnopqrstuvwxyz 50abcdefghijklmnopqrstuvwxyz 51abcdefghijklmnopqrstuvwxyz 52abcdefghijklmnopqrstuvwxyz 53abcdefghijklmnopqrstuvwxyz 54abcdefghijklmnopqrstuvwxyz 55abcdefghijklmnopqrstuvwxyz 56abcdefghijklmnopqrstuvwxyz 57abcdefghijklmnopqrstuvwxyz 58abcdefghijklmnopqrstuvwxyz 59abcdefghijklmnopqrstuvwxyz 60abcdefghijklmnopqrstuvwxyz 61abcdefghijklmnopqrstuvwxyz 62abcdefghijklmnopqrstuvwxyz abcdefghijklmnopqrstuvwxyz 63 abcdefghijklmnopqrstuvwxyz -------------------- 1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz 18abcdefghijklmnopqrstuvwxyz 19abcdefghijklmnopqrstuvwxyz 20abcdefghijklmnopqrstuvwxyz 21abcdefghijklmnopqrstuvwxyz 22abcdefghijklmnopqrstuvwxyz 23abcdefghijklmnopqrstuvwxyz 24abcdefghijklmnopqrstuvwxyz 25abcdefghijklmnopqrstuvwxyz 26abcdefghijklmnopqrstuvwxyz 27abcdefghijklmnopqrstuvwxyz 28abcdefghijklmnopqrstuvwxyz 29abcdefghijklmnopqrstuvwxyz 30abcdefghijklmnopqrstuvwxyz 31abcdefghijklmnopqrstuvwxyz 32abcdefghijklmnopqrstuvwxyz 0123456789a0123456789a0123456789a", + "h": "I AM STILL THE SAME", + "i": "0123456789a0123456789a0123456789a", + "j": "0123456789a0123456789a0123456789a", + "k": "0123456789a0123456789a0123456789a", + "l": "0123456789a0123456789a0123456789a", + "m": "1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz ", + "n": "mock1", +} + diff --git a/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts b/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts index b62d23d2d7..74bde9b562 100644 --- a/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts +++ b/lix/packages/sdk/src/benchmark/changetable-sketch.bench.ts @@ -6,6 +6,13 @@ import { openLixInMemory, type Change, } from "../index.js"; +import { getLeafChangeRecursive } from "../query-utilities/get-leaf-change.js"; +import { sql, type RawBuilder } from "kysely"; +import fs from 'fs' + +function json (object: T): RawBuilder { + return sql`jsonb(${JSON.stringify(object)})` + } const createChange = (parentChangeId?: string) => { samplePaylod.resizingConstraint += 1; @@ -19,11 +26,14 @@ const createChange = (parentChangeId?: string) => { type: "mock", // @ts-expect-error - type error in lix value: JSON.stringify(samplePaylod), + valueB: json(samplePaylod) as any, + commit_id: 'abuse_for_column_in_overflow' }; return change; }; const setupLix = async (nChanges: number) => { + console.log('setting up lix for ' + nChanges + ' changes' ) const lix = await openLixInMemory({ blob: await newLixFile(), }); @@ -40,13 +50,26 @@ const setupLix = async (nChanges: number) => { mockChanges.push(change); } - const batchSize = 256; + const batchSize = 1; // Insert changes in batches for (let i = 0; i < mockChanges.length; i += batchSize) { const batch = mockChanges.slice(i, i + batchSize); + + // console.log(lix.db.insertInto("change").values(batch).compile()) await lix.db.insertInto("change").values(batch).executeTakeFirst(); } // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + console.log('setting up lix for ' + nChanges + ' changes.... done' ) + + + const dbContent = await lix.toBlob() + const ab = await dbContent.arrayBuffer() + const buffer = Buffer.from(ab) + + fs.writeFile('output.db', buffer, (err) => { + if (err) throw err; + console.log('Blob has been written to output.txt'); + }); return { lix, firstChangeId }; }; @@ -68,14 +91,46 @@ for (let i = 0; i < 5; i++) { async () => { let project = await setupLix(nChanges); - bench.skip("payload->property", async () => { + bench("JSON payload-> first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("value", "->>").key("firstProperty"), + "=", + "rectangle", + ) + + .executeTakeFirst(); + + // console.log(result) + + // console.log(result) + }); + + bench("JSONB payload->first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("firstProperty"), + "=", + "rectangle", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench.skip("JSON payload-> last property", async () => { let result = await project.lix.db .selectFrom("change") .selectAll() .where( - (eb) => eb.ref("value", "->>").key("resizingConstraint"), + (eb) => eb.ref("value", "->>").key("lastProperty"), "=", - 100, + "teststring", ) .executeTakeFirst(); @@ -83,7 +138,22 @@ for (let i = 0; i < 5; i++) { // console.log(result) }); - bench.skip("column", async () => { + bench("JSONB payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where( + (eb) => eb.ref("valueB", "->>").key("lastProperty"), + "=", + "teststring", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column", async () => { let result = await project.lix.db .selectFrom("change") .selectAll() @@ -94,26 +164,24 @@ for (let i = 0; i < 5; i++) { // console.log(result) }); - bench.skip( - "getLeafNode", - async () => { - await getLeafChange({ - lix: project.lix, - change: { id: project.firstChangeId } as Change, - }); + bench("column in overflow", async () => { + let result = await project.lix.db + .selectFrom("change") + .selectAll() + .where("commit_id", "=", "abuse_for_column_in_overflow") - // console.log(result) - }, - { - iterations: 2, - }, - ); - }, + .executeTakeFirst(); + + // console.log(result) + }); + + + } ); } const samplePaylod = { - _class: "rectangle", + firstProperty: "rectangle", do_objectID: "AEB63D36-FA90-4F68-90B0-BFEC297D1AF0", booleanOperation: -1, isFixedToViewport: false, @@ -283,7 +351,7 @@ const samplePaylod = { point: "{1, 1}", }, { - _class: "curvePoint", + _class: "cur", cornerRadius: 10, cornerStyle: 0, curveFrom: "{0, 1}", @@ -297,4 +365,5 @@ const samplePaylod = { fixedRadius: 0, needsConvertionToNewRoundCorners: false, hasConvertedToNewRoundCorners: true, + lastProperty: "teststring" }; diff --git a/lix/packages/sdk/src/benchmark/changetable-subsetresult.bench.ts b/lix/packages/sdk/src/benchmark/changetable-subsetresult.bench.ts new file mode 100644 index 0000000000..3132b5ff80 --- /dev/null +++ b/lix/packages/sdk/src/benchmark/changetable-subsetresult.bench.ts @@ -0,0 +1,204 @@ +import { v4 } from "uuid"; +import { afterEach, beforeEach, bench, describe } from "vitest"; +import { + getLeafChange, + newLixFile, + openLixInMemory, + type Change, +} from "../index.js"; +import { getLeafChangeRecursive } from "../query-utilities/get-leaf-change.js"; +import { sql, type RawBuilder } from "kysely"; +import fs from 'fs' + +function json (object: T): RawBuilder { + return sql`jsonb(${JSON.stringify(object)})` + } + +const createChange = (parentChangeId?: string) => { + + const change: Change = { + id: v4(), + parent_id: parentChangeId, + operation: "create", + file_id: "mock", + plugin_key: parentChangeId ? "mock" : "mock1", + type: "mock", + // @ts-expect-error - type error in lix + value: JSON.stringify(samplePaylod), + valueB: json(samplePaylod) as any, + commit_id: 'abuse_for_column_in_overflow' + }; + return change; +}; + +const setupLix = async (nChanges: number) => { + console.log('setting up lix for ' + nChanges + ' changes' ) + const lix = await openLixInMemory({ + blob: await newLixFile(), + }); + + await sql`PRAGMA page_size = 16384;`.execute(lix.db) + + const mockChanges: Change[] = []; + let lastChangeId: undefined | string = undefined; + let firstChangeId: undefined | string = undefined; + for (let i = 0; i < nChanges; i++) { + const change = createChange(lastChangeId); + lastChangeId = change.id; + if (!firstChangeId) { + firstChangeId = change.id; + } + mockChanges.push(change); + } + + const batchSize = 1; + // Insert changes in batches + for (let i = 0; i < mockChanges.length; i += batchSize) { + const batch = mockChanges.slice(i, i + batchSize); + + // console.log(lix.db.insertInto("change").values(batch).compile()) + await lix.db.insertInto("change").values(batch).executeTakeFirst(); + } + // await lix.db.insertInto("change").values(mockChanges).executeTakeFirst(); + console.log('setting up lix for ' + nChanges + ' changes.... done' ) + + + const dbContent = await lix.toBlob() + const ab = await dbContent.arrayBuffer() + const buffer = Buffer.from(ab) + + fs.writeFile('output.db', buffer, (err) => { + if (err) throw err; + console.log('Blob has been written to output.txt'); + }); + + return { lix, firstChangeId }; +}; + +async function wait(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); // 1000 milliseconds = 1 second + }); +} + +for (let i = 0; i < 5; i++) { + const nChanges = Math.pow(10, i); + describe( + "select changes via JSON / JSONB / COLUMN (last and first) " + + nChanges + + " changes", + async () => { + let project = await setupLix(nChanges); + + bench("JSON payload-> first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .select('id') + .where( + (eb) => eb.ref("value", "->>").key("a"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + + // console.log(result) + }); + + + + bench("JSON payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .select('id') + .where( + (eb) => eb.ref("value", "->>").key("n"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + + bench("JSONB payload->first property", async () => { + let result = await project.lix.db + .selectFrom("change") + .select('id') + .where( + (eb) => eb.ref("valueB", "->>").key("a"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("JSONB payload-> last property", async () => { + let result = await project.lix.db + .selectFrom("change") + .select('id') + .where( + (eb) => eb.ref("valueB", "->>").key("n"), + "=", + "mock1", + ) + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column", async () => { + let result = await project.lix.db + .selectFrom("change") + .select('id') + .where("plugin_key", "=", "mock1") + + .executeTakeFirst(); + + // console.log(result) + }); + + bench("column after json", async () => { + let result = await project.lix.db + .selectFrom("change") + .select('id') + .where("commit_id", "=", "abuse_for_column_in_overflow") + + .executeTakeFirst(); + + // console.log(result) + }); + + + } + ); +} + +const samplePaylod = { + "z": "1", + "a": "mock1", + "b": "1", + "c": "1", + "d": "1", + "e": "1", + "f": "1", + "g": "1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz 18abcdefghijklmnopqrstuvwxyz 19abcdefghijklmnopqrstuvwxyz 20abcdefghijklmnopqrstuvwxyz 21abcdefghijklmnopqrstuvwxyz 22abcdefghijklmnopqrstuvwxyz 23abcdefghijklmnopqrstuvwxyz 24abcdefghijklmnopqrstuvwxyz 25abcdefghijklmnopqrstuvwxyz 26abcdefghijklmnopqrstuvwxyz 27abcdefghijklmnopqrstuvwxyz 28abcdefghijklmnopqrstuvwxyz 29abcdefghijklmnopqrstuvwxyz 30abcdefghijklmnopqrstuvwxyz 31abcdefghijklmnopqrstuvwxyz 32abcdefghijklmnopqrstuvwxyz 33abcdefghijklmnopqrstuvwxyz 34abcdefghijklmnopqrstuvwxyz 35abcdefghijklmnopqrstuvwxyz 36abcdefghijklmnopqrstuvwxyz 37abcdefghijklmnopqrstuvwxyz 38abcdefghijklmnopqrstuvwxyz 39abcdefghijklmnopqrstuvwxyz 40abcdefghijklmnopqrstuvwxyz 41abcdefghijklmnopqrstuvwxyz 42abcdefghijklmnopqrstuvwxyz 43abcdefghijklmnopqrstuvwxyz 44abcdefghijklmnopqrstuvwxyz 45abcdefghijklmnopqrstuvwxyz 46abcdefghijklmnopqrstuvwxyz 47abcdefghijklmnopqrstuvwxyz 48abcdefghijklmnopqrstuvwxyz 49abcdefghijklmnopqrstuvwxyz 50abcdefghijklmnopqrstuvwxyz 51abcdefghijklmnopqrstuvwxyz 52abcdefghijklmnopqrstuvwxyz 53abcdefghijklmnopqrstuvwxyz 54abcdefghijklmnopqrstuvwxyz 55abcdefghijklmnopqrstuvwxyz 56abcdefghijklmnopqrstuvwxyz 57abcdefghijklmnopqrstuvwxyz 58abcdefghijklmnopqrstuvwxyz 59abcdefghijklmnopqrstuvwxyz 60abcdefghijklmnopqrstuvwxyz 61abcdefghijklmnopqrstuvwxyz 62abcdefghijklmnopqrstuvwxyz abcdefghijklmnopqrstuvwxyz 63 abcdefghijklmnopqrstuvwxyz -------------------- 1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz 18abcdefghijklmnopqrstuvwxyz 19abcdefghijklmnopqrstuvwxyz 20abcdefghijklmnopqrstuvwxyz 21abcdefghijklmnopqrstuvwxyz 22abcdefghijklmnopqrstuvwxyz 23abcdefghijklmnopqrstuvwxyz 24abcdefghijklmnopqrstuvwxyz 25abcdefghijklmnopqrstuvwxyz 26abcdefghijklmnopqrstuvwxyz 27abcdefghijklmnopqrstuvwxyz 28abcdefghijklmnopqrstuvwxyz 29abcdefghijklmnopqrstuvwxyz 30abcdefghijklmnopqrstuvwxyz 31abcdefghijklmnopqrstuvwxyz 32abcdefghijklmnopqrstuvwxyz 0123456789a0123456789a0123456789a", + "h": "I AM STILL THE SAME", + "i": "0123456789a0123456789a0123456789a", + "j": "0123456789a0123456789a0123456789a", + "k": "0123456789a0123456789a0123456789a", + "l": "0123456789a0123456789a0123456789a", + "m": "1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz1abcdefghijklmnopqrstuvwxyz 2abcdefghijklmnopqrstuvwxyz 3abcdefghijklmnopqrstuvwxyz 4abcdefghijklmnopqrstuvwxyz 5abcdefghijklmnopqrstuvwxyz 6abcdefghijklmnopqrstuvwxyz 7abcdefghijklmnopqrstuvwxyz 8abcdefghijklmnopqrstuvwxyz 9abcdefghijklmnopqrstuvwxyz 10abcdefghijklmnopqrstuvwxyz 11abcdefghijklmnopqrstuvwxyz 12abcdefghijklmnopqrstuvwxyz 13abcdefghijklmnopqrstuvwxyz 14abcdefghijklmnopqrstuvwxyz 15abcdefghijklmnopqrstuvwxyz 16abcdefghijklmnopqrstuvwxyz 17abcdefghijklmnopqrstuvwxyz ", + "n": "mock1", +} + diff --git a/lix/packages/sdk/src/database/createSchema.ts b/lix/packages/sdk/src/database/createSchema.ts index 24a147e828..48ccc1da4c 100644 --- a/lix/packages/sdk/src/database/createSchema.ts +++ b/lix/packages/sdk/src/database/createSchema.ts @@ -36,11 +36,14 @@ export async function createSchema(args: { db: Kysely }) { plugin_key TEXT NOT NULL, operation TEXT NOT NULL, value TEXT, + valueB BLOB, + meta TEXT, commit_id TEXT, created_at TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL ) strict; + CREATE INDEX IF NOT EXISTS idx_change_parent_id ON change (parent_id); CREATE TABLE conflict ( change_id TEXT NOT NULL, conflicting_change_id TEXT NOT NULL, diff --git a/lix/packages/sdk/src/database/schema.ts b/lix/packages/sdk/src/database/schema.ts index 2cb3de6be4..f312554d33 100644 --- a/lix/packages/sdk/src/database/schema.ts +++ b/lix/packages/sdk/src/database/schema.ts @@ -111,6 +111,8 @@ type ChangeTable = { * - For an inlang message change, the value would be the new message. */ value?: Record & { id: string }; + + valueB?: Record & { id: string }; /** * Additional metadata for the change used by the plugin * to process changes. diff --git a/lix/packages/sdk/src/query-utilities/get-leaf-change.ts b/lix/packages/sdk/src/query-utilities/get-leaf-change.ts index d1eacb34ad..b5281b006e 100644 --- a/lix/packages/sdk/src/query-utilities/get-leaf-change.ts +++ b/lix/packages/sdk/src/query-utilities/get-leaf-change.ts @@ -1,4 +1,5 @@ import type { Change, LixReadonly } from "@lix-js/sdk"; +import { sql } from "kysely"; /** * Find the last "child" change of the given change. @@ -27,3 +28,48 @@ export async function getLeafChange(args: { return nextChange; } + +/** + * Find the last "child" change of the given change. + */ +export async function getLeafChangeRecursive(args: { + change: Change; + lix: LixReadonly; +}): Promise { + + const statement = args.lix.db + .withRecursive("ChangeGraph", (re) => { + return re + .selectFrom('change') + .selectAll() + .select(({ fn, val, ref }) => [ + sql`1`.as("row_counter")]) + .where("id", "=", args.change.id) + .unionAll((unionAll) => { + return unionAll.selectFrom("change as c").innerJoin("ChangeGraph as cg", "cg.id", "c.parent_id").selectAll("c").select(({ fn, val, ref }) => [ + sql`cg.row_counter + 1`.as("row_counter")]); + }) + }) + .selectFrom("ChangeGraph") + .selectAll() + .orderBy("row_counter", "desc") + .limit(1) + + + return (await statement.executeTakeFirst()) ?? args.change; + + // const _true = true; + + // let nextChange = args.change; + + // while (_true) { + + // if (!childChange) { + // break; + // } + + // nextChange = childChange; + // } + + // return nextChange; +} diff --git a/lix/packages/sdk/src/types.ts b/lix/packages/sdk/src/types.ts index 2441162a73..bdc06bd127 100644 --- a/lix/packages/sdk/src/types.ts +++ b/lix/packages/sdk/src/types.ts @@ -2,9 +2,9 @@ import type { openLix } from "./open/openLix.js"; export type Lix = Awaited>; - export type LixReadonly = Pick & { db: { selectFrom: Lix["db"]["selectFrom"]; + withRecursive: Lix["db"]["withRecursive"]; }; };